diff --git a/.github/actions/dynamatrix/action.yml b/.github/actions/dynamatrix/action.yml new file mode 100644 index 00000000..11eef3bb --- /dev/null +++ b/.github/actions/dynamatrix/action.yml @@ -0,0 +1,22 @@ +name: Create matrix +description: Create matrix +inputs: + matrix_yaml: + description: input yaml matrix as multiline string; any entry with a bool true `omit` key will be filtered from the output matrix + required: true +outputs: + matrix_json: + description: filtered matrix as JSON + value: ${{ steps.matrix_gen.outputs.matrix_json }} + +runs: + using: "composite" + + steps: + - id: matrix_gen + run: | + # FIXME: input sanity check to prevent shell injection + python3 $GITHUB_ACTION_PATH/matrix_yaml_to_json.py --from-stdin << EOF + ${{ inputs.matrix_yaml }} + EOF + shell: bash diff --git a/.github/actions/dynamatrix/matrix_yaml_to_json.py b/.github/actions/dynamatrix/matrix_yaml_to_json.py new file mode 100644 index 00000000..77bebaf4 --- /dev/null +++ b/.github/actions/dynamatrix/matrix_yaml_to_json.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import sys +import typing as t +import yaml + +from collections.abc import MutableMapping, Sequence + +skipped_entries = [] + +def _filter_omit_entries(value): + if isinstance(value, MutableMapping): + if (omit_value := value.pop('omit', ...)) is not ...: + if omit_value is True or str(omit_value).lower().strip() == 'true': + print(f'omitting {value} from matrix') + skipped_entries.append(value) + return ... + + return {k: v for k, v in ((k, _filter_omit_entries(v)) for k, v in value.items()) if v is not ...} + + if isinstance(value, str): + return value + + if isinstance(value, Sequence): + return [v for v in (_filter_omit_entries(v) for v in value) if v is not ...] + + return value + +def main(): + p = argparse.ArgumentParser(description='GHA YAML matrix filter') + required_grp = p.add_mutually_exclusive_group(required=True) + required_grp.add_argument('--from-stdin', action='store_true', help='read input YAML from stdin') + required_grp.add_argument('--from-file', type=pathlib.Path, help='read input YAML from file path') + + args = p.parse_args() + + path: pathlib.Path | None + + matrix_yaml: str + + if path := args.from_file: + matrix_yaml = path.read_text() + elif args.from_stdin: + matrix_yaml = sys.stdin.read() + else: + raise Exception('no source provided for matrix yaml') + + raw_matrix = yaml.safe_load(matrix_yaml) + filtered_matrix = _filter_omit_entries(raw_matrix) + + output_matrix_json = json.dumps(filtered_matrix) + output_skipped_matrix_json = json.dumps(skipped_entries) + + print(f'filtered matrix: {output_matrix_json}') + print(f'skipped entries: {output_skipped_matrix_json}') + + if (gh_output := os.environ.get('GITHUB_OUTPUT')): + print('setting step output var matrix_json; skipped_matrix_json...') + with pathlib.Path(gh_output).open('a') as env_fd: + env_fd.write(f'matrix_json<<__MATRIX_EOF\n{output_matrix_json}\n__MATRIX_EOF\n') + env_fd.write(f'skipped_matrix_json<<__MATRIX_EOF\n{output_skipped_matrix_json}\n__MATRIX_EOF\n') + else: + print("GITHUB_OUTPUT not set; skipping variable output") + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c823062a..28800091 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,8 +1,4 @@ --- -# This is the CI workflow (not the artifact build/release workflow). The workflows -# are split because GHA doesn't support a dynamic/conditional matrix. This workflow -# has slow jobs and jobs that require private GHA runners (eg, M1 Mac) commented out. -# Ensure changes are synced with manual_artifact_build.yaml. name: PyYAML CI on: @@ -10,25 +6,52 @@ on: pull_request: types: [opened, synchronize, reopened] workflow_dispatch: + inputs: + libyaml_repo: + description: Repo URL for libyaml + type: string + default: https://github.com/yaml/libyaml + libyaml_ref: + description: Tag/branch ref for libyaml + type: string + default: 0.2.5 + skip_artifact_upload: + description: Skip most artifact uploads? + type: boolean + default: true + skip_ci_redundant_jobs: + description: Skip redundant jobs for CI? + type: boolean + default: true + skip_slow_jobs: + description: Skip slow (emulated) jobs for CI? + type: boolean + default: true env: - LIBYAML_REPO: https://github.com/yaml/libyaml - LIBYAML_REF: 0.2.5 + LIBYAML_REPO: ${{ inputs.libyaml_repo || 'https://github.com/yaml/libyaml' }} # FIXME: can we ref the input.default value? + LIBYAML_REF: ${{ inputs.libyaml_ref || '0.2.5' }} # FIXME: can we ref the input.default value? + skip_ci_redundant_jobs: ${{ inputs.skip_ci_redundant_jobs || github.event_name == 'pull_request' || github.event_name == 'push' }} + skip_slow_jobs: ${{ inputs.skip_slow_jobs || github.event_name == 'pull_request' || github.event_name == 'push' }} + skip_artifact_upload: ${{ (github.event_name == 'workflow_dispatch' && inputs.skip_artifact_upload) || github.event_name != 'workflow_dispatch' }} jobs: python_sdist: name: pyyaml sdist - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 + outputs: + artifact_name: ${{ steps.build_sdist.outputs.artifact_name }} steps: - name: Checkout PyYAML - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install a python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.x - name: Build sdist + id: build_sdist env: PYYAML_FORCE_CYTHON: 1 PYYAML_FORCE_LIBYAML: 0 @@ -36,64 +59,60 @@ jobs: python -V python -m pip install build - python -m build . - - # Ensure exactly one artifact was produced. - [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || { - echo "Unexpected content in dist dir: '$(ls dist/*.tar.gz)'." - exit 1 - } - - - name: Test sdist - run: | - # Install some libyaml headers. - # TODO Should we smoke test the sdist against the libyaml we built? - sudo apt update - sudo apt install libyaml-dev -y - - # Ensure Cython is not present so we use only what's in the sdist. - python -m pip uninstall Cython -y || true + python -m build -s . - # Pass no extra args. - # We should auto-install with libyaml since it's present. - python -m pip install dist/*.tar.gz -v - - python packaging/build/smoketest.py + echo "artifact_name=$(ls ./dist)" >> "$GITHUB_OUTPUT" - name: Upload sdist artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: dist - path: dist/*.tar.gz + name: ${{ steps.build_sdist.outputs.artifact_name }} + path: dist/${{ steps.build_sdist.outputs.artifact_name }} + if-no-files-found: error + # always upload the sdist artifact- all the wheel build jobs require it + make_linux_libyaml_matrix: + runs-on: ubuntu-22.04 + outputs: + matrix_json: ${{ steps.make_matrix.outputs.matrix_json }} + steps: + - uses: actions/checkout@v4 + - name: make a matrix + id: make_matrix + uses: ./.github/actions/dynamatrix + with: + matrix_yaml: | + include: + - { platform: manylinux1, arch: x86_64 } + - { platform: manylinux2014, arch: x86_64 } + - { platform: manylinux2014, arch: aarch64, omit: ${{ env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, omit: ${{ env.skip_slow_jobs }} } + - { platform: musllinux_1_1, arch: x86_64 } + - { platform: musllinux_1_1, arch: aarch64, omit: ${{ env.skip_slow_jobs }} } linux_libyaml: - name: libyaml ${{matrix.cfg.arch}} ${{matrix.cfg.platform}} - runs-on: ubuntu-latest + needs: [make_linux_libyaml_matrix] + name: libyaml ${{ matrix.platform }} ${{ matrix.arch }} + runs-on: ubuntu-22.04 strategy: - matrix: - cfg: - - { platform: manylinux1, arch: x86_64 } - - { platform: manylinux2014, arch: x86_64 } -# - { platform: manylinux2014, arch: aarch64 } -# - { platform: manylinux2014, arch: s390x } - - { platform: musllinux_1_1, arch: x86_64 } + fail-fast: false + matrix: ${{ fromJSON(needs.make_linux_libyaml_matrix.outputs.matrix_json) }} env: - DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}} + DOCKER_IMAGE: quay.io/pypa/${{ matrix.platform }}_${{ matrix.arch }} steps: - name: Check cached libyaml state id: cached_libyaml - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: libyaml - key: libyaml_${{matrix.cfg.platform}}_${{matrix.cfg.arch}}_${{env.LIBYAML_REF}} + key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} - name: configure docker foreign arch support - uses: docker/setup-qemu-action@v2 - if: matrix.cfg.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true' + uses: docker/setup-qemu-action@v3 + if: matrix.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true' - name: Checkout pyyaml - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: steps.cached_libyaml.outputs.cache-hit != 'true' - name: Build libyaml @@ -112,58 +131,78 @@ jobs: sudo chmod -R a+r ./libyaml/ if: steps.cached_libyaml.outputs.cache-hit != 'true' + make_linux_pyyaml_matrix: + runs-on: ubuntu-22.04 + outputs: + matrix_json: ${{ steps.make_matrix.outputs.matrix_json }} + steps: + - uses: actions/checkout@v4 + - name: make a matrix + id: make_matrix + uses: ./.github/actions/dynamatrix + with: + matrix_yaml: | + include: + - { platform: manylinux1, arch: x86_64, spec: cp38 } + - { platform: manylinux1, arch: x86_64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: manylinux2014, arch: x86_64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: manylinux2014, arch: x86_64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: manylinux2014, arch: x86_64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: manylinux2014, arch: x86_64, spec: cp313 } + - { platform: manylinux2014, arch: aarch64, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: aarch64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: aarch64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: aarch64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: aarch64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: aarch64, spec: cp313, omit: ${{ env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, spec: cp313, omit: ${{ env.skip_slow_jobs }} } + - { platform: musllinux_1_1, arch: x86_64, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: musllinux_1_1, arch: x86_64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: musllinux_1_1, arch: x86_64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: musllinux_1_1, arch: x86_64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: musllinux_1_1, arch: x86_64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: musllinux_1_1, arch: x86_64, spec: cp313 } + - { platform: musllinux_1_1, arch: aarch64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: musllinux_1_1, arch: aarch64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: musllinux_1_1, arch: aarch64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: musllinux_1_1, arch: aarch64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: musllinux_1_1, arch: aarch64, spec: cp313, omit: ${{ env.skip_slow_jobs }} } + linux_pyyaml: - needs: linux_libyaml - name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.spec}} - runs-on: ubuntu-latest + needs: [python_sdist, linux_libyaml, make_linux_pyyaml_matrix] + name: pyyaml ${{matrix.spec}}-${{matrix.platform}}_${{matrix.arch}} + runs-on: ubuntu-22.04 strategy: - matrix: - include: - - { platform: manylinux1, arch: x86_64, spec: cp36 } -# - { platform: manylinux1, arch: x86_64, spec: cp37 } -# - { platform: manylinux1, arch: x86_64, spec: cp38 } -# - { platform: manylinux1, arch: x86_64, spec: cp39 } -# - { platform: manylinux2014, arch: x86_64, spec: cp310 } -# - { platform: manylinux2014, arch: x86_64, spec: cp311 } - - { platform: manylinux2014, arch: x86_64, spec: cp312 } -# - { platform: manylinux2014, arch: aarch64, spec: cp36 } -# - { platform: manylinux2014, arch: aarch64, spec: cp37 } -# - { platform: manylinux2014, arch: aarch64, spec: cp38 } -# - { platform: manylinux2014, arch: aarch64, spec: cp39 } -# - { platform: manylinux2014, arch: aarch64, spec: cp310 } -# - { platform: manylinux2014, arch: aarch64, spec: cp311 } -# - { platform: manylinux2014, arch: aarch64, spec: cp312 } -# - { platform: manylinux2014, arch: s390x, spec: cp36 } -# - { platform: manylinux2014, arch: s390x, spec: cp37 } -# - { platform: manylinux2014, arch: s390x, spec: cp38 } -# - { platform: manylinux2014, arch: s390x, spec: cp39 } -# - { platform: manylinux2014, arch: s390x, spec: cp310 } -# - { platform: manylinux2014, arch: s390x, spec: cp311 } -# - { platform: manylinux2014, arch: s390x, spec: cp312 } -# - { platform: musllinux_1_1, arch: x86_64, spec: cp38 } -# - { platform: musllinux_1_1, arch: x86_64, spec: cp39 } -# - { platform: musllinux_1_1, arch: x86_64, spec: cp310 } -# - { platform: musllinux_1_1, arch: x86_64, spec: cp311 } - - { platform: musllinux_1_1, arch: x86_64, spec: cp312 } + fail-fast: false + matrix: ${{ fromJSON(needs.make_linux_pyyaml_matrix.outputs.matrix_json) }} steps: - - name: Checkout PyYAML - uses: actions/checkout@v3 + - name: fetch sdist artifact + id: fetch_sdist + uses: actions/download-artifact@v4 + with: + name: ${{ needs.build_sdist.outputs.artifact_name }} - name: Fetch cached libyaml id: cached_libyaml - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: libyaml key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} fail-on-cache-miss: true - name: configure docker foreign arch support - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 if: matrix.arch != 'x86_64' - name: Build/Test/Package + id: build env: CIBW_ARCHS: all # HACK: ick, maybe deconstruct the matrix a bit or query cibuildwheel for its default target *linux spec first? @@ -171,24 +210,47 @@ jobs: CIBW_BUILD_VERBOSITY: 1 # containerized Linux builds require explicit CIBW_ENVIRONMENT CIBW_ENVIRONMENT: > - C_INCLUDE_PATH=libyaml/include - LIBRARY_PATH=libyaml/src/.libs - LD_LIBRARY_PATH=libyaml/src/.libs + LD_LIBRARY_PATH=../libyaml/src/.libs PYYAML_FORCE_CYTHON=1 PYYAML_FORCE_LIBYAML=1 - CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_img || '' }} + CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_img || '' }} + CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux_img || '' }} + CIBW_MANYLINUX_S390X_IMAGE: ${{ matrix.manylinux_img || '' }} + CIBW_MUSLLINUX_X86_64_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_1' }} + CIBW_MUSLLINUX_I686_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_1' }} + CIBW_MUSLLINUX_AARCH64_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_1' }} + CIBW_PRERELEASE_PYTHONS: 1 + CIBW_TEST_COMMAND: cd {project}; pytest + CIBW_TEST_REQUIRES: pytest run: | set -eux + python3 -V - python3 -m pip install -U --user cibuildwheel - python3 -m cibuildwheel --platform auto --output-dir dist . + python3 -m pip install -U --user ${{ matrix.cibw_version || 'cibuildwheel' }} + + mkdir pyyaml + + tar zxf ${{ steps.fetch_sdist.outputs.download-path }}/pyyaml*.tar.gz/pyyaml*.tar.gz --strip-components=1 -C pyyaml + + cat << 'EOF' > build_config.toml + [tool.cibuildwheel.config-settings] + pyyaml_build_config='{"force":1, "library_dirs": ["../libyaml/src/.libs"], "include_dirs": ["../libyaml/include"]}' + EOF + + CIBW_BEFORE_BUILD="ls -l {project}" python3 -m cibuildwheel --config-file $(pwd)/build_config.toml --platform auto --output-dir ./dist ./pyyaml + + echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT" + # FIXME: ensure exactly one artifact - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: dist + name: ${{ steps.build.outputs.artifact_name }} path: dist/*.whl if-no-files-found: error + if: ${{ env.skip_artifact_upload != 'true' }} + macos_libyaml: name: libyaml macos ${{matrix.arch}} @@ -196,25 +258,25 @@ jobs: matrix: include: - arch: x86_64 -# - arch: arm64 -# runs_on: [self-hosted, macOS, arm64] -# deployment_target: '11.0' -# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} -# sdkroot: macosx11.3 + runs-on: macos-13 + run_wrapper: arch -x86_64 bash --noprofile --norc -eo pipefail {0} + - arch: arm64 + deployment_target: '11.0' + run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} defaults: run: - shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }} - runs-on: ${{ matrix.runs_on || 'macos-11' }} + shell: ${{ matrix.run_wrapper || 'arch -x86_64 bash --noprofile --norc -eo pipefail {0}' }} + runs-on: ${{ matrix.runs-on || 'macos-14' }} steps: - name: Check cached libyaml state id: cached_libyaml - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: libyaml key: libyaml_macos_${{matrix.arch}}_${{env.LIBYAML_REF}} - name: Checkout PyYAML - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: steps.cached_libyaml.outputs.cache-hit != 'true' - name: Build libyaml @@ -223,112 +285,154 @@ jobs: SDKROOT: ${{ matrix.sdkroot || 'macosx' }} run: | set -eux - brew install automake coreutils m4 + brew install automake coreutils m4 libtool bash ./packaging/build/libyaml.sh echo "finished artifact arch is $(lipo -archs libyaml/src/.libs/libyaml.a)" if: steps.cached_libyaml.outputs.cache-hit != 'true' + make_macos_pyyaml_matrix: + runs-on: ubuntu-22.04 + outputs: + matrix_json: ${{ steps.make_matrix.outputs.matrix_json }} + steps: + - uses: actions/checkout@v4 + - name: make a matrix + id: make_matrix + uses: ./.github/actions/dynamatrix + with: + matrix_yaml: | + include: + - spec: cp38-macosx_x86_64 + cibw_version: cibuildwheel==2.11.1 + runs_on: [macos-13] + omit: ${{ env.skip_ci_redundant_jobs }} + - spec: cp39-macosx_x86_64 + runs_on: [macos-13] + omit: ${{ env.skip_ci_redundant_jobs }} + - spec: cp310-macosx_x86_64 + runs_on: [macos-13] + omit: ${{ env.skip_ci_redundant_jobs }} + - spec: cp311-macosx_x86_64 + runs_on: [macos-13] + omit: ${{ env.skip_ci_redundant_jobs }} + - spec: cp312-macosx_x86_64 + runs_on: [macos-13] + omit: ${{ env.skip_ci_redundant_jobs }} + - spec: cp313-macosx_x86_64 + runs_on: [macos-13] + + - spec: cp39-macosx_arm64 + deployment_target: '11.0' + arch: arm64 + omit: ${{ env.skip_ci_redundant_jobs }} + + - spec: cp310-macosx_arm64 + deployment_target: '11.0' + arch: arm64 + omit: ${{ env.skip_ci_redundant_jobs }} + + - spec: cp311-macosx_arm64 + deployment_target: '11.0' + arch: arm64 + omit: ${{ env.skip_ci_redundant_jobs }} + + - spec: cp312-macosx_arm64 + deployment_target: '11.0' + arch: arm64 + omit: ${{ env.skip_ci_redundant_jobs }} + + - spec: cp313-macosx_arm64 + deployment_target: '11.0' + arch: arm64 + macos_pyyaml: - needs: macos_libyaml + needs: [python_sdist, macos_libyaml, make_macos_pyyaml_matrix] name: pyyaml ${{ matrix.spec }} - runs-on: ${{ matrix.runs_on || 'macos-11' }} + runs-on: ${{ matrix.runs_on || 'macos-14' }} defaults: run: shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }} strategy: - matrix: - include: -# - spec: cp36-macosx_x86_64 -# cibw_version: cibuildwheel==2.11.1 -# - spec: cp37-macosx_x86_64 -# cibw_version: cibuildwheel==2.11.1 -# - spec: cp38-macosx_x86_64 -# cibw_version: cibuildwheel==2.11.1 -# - spec: cp39-macosx_x86_64 -# - spec: cp310-macosx_x86_64 -# - spec: cp311-macosx_x86_64 - - spec: cp312-macosx_x86_64 - -# # build for arm64 under a hacked macOS 12 self-hosted x86_64-on-arm64 runner until arm64 is fully supported -# - spec: cp39-macosx_arm64 -# deployment_target: '11.0' -# runs_on: [self-hosted, macOS, arm64] -# arch: arm64 -# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} -# sdkroot: macosx11.3 -# -# - spec: cp310-macosx_arm64 -# deployment_target: '11.0' -# runs_on: [self-hosted, macOS, arm64] -# arch: arm64 -# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} -# sdkroot: macosx11.3 -# -# - spec: cp311-macosx_arm64 -# deployment_target: '11.0' -# runs_on: [self-hosted, macOS, arm64] -# arch: arm64 -# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} -# sdkroot: macosx11.3 -# -# - spec: cp312-macosx_arm64 -# deployment_target: '11.0' -# runs_on: [self-hosted, macOS, arm64] -# arch: arm64 -# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} -# sdkroot: macosx11.3 - + fail-fast: false + matrix: ${{ fromJSON(needs.make_macos_pyyaml_matrix.outputs.matrix_json) }} steps: - - name: Checkout PyYAML - uses: actions/checkout@v3 + - name: fetch sdist artifact + id: fetch_sdist + uses: actions/download-artifact@v4 + with: + name: ${{ needs.build_sdist.outputs.artifact_name }} - name: Get cached libyaml state id: cached_libyaml - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: libyaml key: libyaml_macos_${{ matrix.arch || 'x86_64' }}_${{env.LIBYAML_REF}} fail-on-cache-miss: true + - name: Install python + uses: actions/setup-python@v5 + with: + python-version: '3.11' # as of 2024-05, this has to be < 3.12 since the macos-13 runner image's + # built-in virtualenv/pip are pinned to busted versions that fail on newer Pythons + - name: Build/Test/Package + id: build env: - C_INCLUDE_PATH: libyaml/include + C_INCLUDE_PATH: ../libyaml/include CIBW_BUILD: ${{matrix.spec}} CIBW_BUILD_VERBOSITY: 1 - CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py - LIBRARY_PATH: libyaml/src/.libs + CIBW_PRERELEASE_PYTHONS: 1 + CIBW_TEST_COMMAND: pytest {package} + CIBW_TEST_REQUIRES: pytest + LIBRARY_PATH: ../libyaml/src/.libs MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }} + PYYAML_FORCE_CYTHON: 1 + PYYAML_FORCE_LIBYAML: 1 SDKROOT: ${{ matrix.sdkroot || 'macosx' }} run: | + set -eux python3 -V python3 -m pip install -U --user ${{ matrix.cibw_version || 'cibuildwheel' }} - python3 -m cibuildwheel --platform auto --output-dir dist . + mkdir pyyaml + + tar zxf pyyaml*.tar.gz/pyyaml*.tar.gz --strip-components=1 -C pyyaml + + cat << 'EOF' > build_config.toml + [tool.cibuildwheel.config-settings] + pyyaml_build_config='{"force":1, "library_dirs": ["../libyaml/src/.libs"], "include_dirs": ["../libyaml/include"]}' + EOF + + python3 -m cibuildwheel --config-file $(pwd)/build_config.toml --platform auto --output-dir ./dist ./pyyaml + + echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT" + # FIXME: ensure exactly one artifact - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: dist + name: ${{ steps.build.outputs.artifact_name }} path: dist/*.whl if-no-files-found: error + if: ${{ env.skip_artifact_upload != 'true' }} + windows_libyaml: - name: libyaml ${{matrix.platform}} ${{matrix.arch}} - runs-on: ${{matrix.platform}} + name: libyaml windows ${{ matrix.arch }} + runs-on: ${{ matrix.platform || 'windows-2022' }} strategy: matrix: include: - - platform: windows-2019 - arch: x64 - - platform: windows-2019 - arch: win32 + - arch: x64 + - arch: win32 steps: - name: Get cached libyaml state id: cached_libyaml - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: libyaml - key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} + key: libyaml_${{ 'windows' }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }} - name: Build libyaml shell: bash @@ -348,74 +452,63 @@ jobs: mkdir libyaml/build pushd libyaml/build - cmake.exe -G "Visual Studio 16 2019" -A ${{ matrix.arch }} -DYAML_STATIC_LIB_NAME=yaml .. + cmake.exe -G "Visual Studio 17 2022" -A ${{ matrix.arch }} -DYAML_STATIC_LIB_NAME=yaml .. cmake.exe --build . --config Release popd + make_windows_pyyaml_matrix: + runs-on: ubuntu-22.04 + outputs: + matrix_json: ${{ steps.make_matrix.outputs.matrix_json }} + steps: + - uses: actions/checkout@v4 + - name: make a matrix + id: make_matrix + uses: ./.github/actions/dynamatrix + with: + matrix_yaml: | + include: + - spec: cp38-win_amd64 + + - spec: cp39-win_amd64 + omit: ${{ env.skip_ci_redundant_jobs }} + + - spec: cp310-win_amd64 + omit: ${{ env.skip_ci_redundant_jobs }} + + - spec: cp311-win_amd64 + omit: ${{ env.skip_ci_redundant_jobs }} + + - spec: cp312-win_amd64 + omit: ${{ env.skip_ci_redundant_jobs }} + + - spec: cp313-win_amd64 + + - spec: cp38-win32 + omit: ${{ env.skip_ci_redundant_jobs }} + + - spec: cp39-win32 + omit: ${{ env.skip_ci_redundant_jobs }} + + - spec: cp310-win32 + omit: ${{ env.skip_ci_redundant_jobs }} + + - spec: cp311-win32 + omit: ${{ env.skip_ci_redundant_jobs }} + + - spec: cp312-win32 + omit: ${{ env.skip_ci_redundant_jobs }} + + - spec: cp313-win32 + omit: ${{ env.skip_ci_redundant_jobs }} windows_pyyaml: - needs: windows_libyaml - name: pyyaml ${{ matrix.platform }} ${{matrix.python_arch}} ${{matrix.spec}} - runs-on: ${{matrix.platform}} + needs: [python_sdist, windows_libyaml, make_windows_pyyaml_matrix] + name: pyyaml ${{matrix.spec}} + runs-on: ${{ matrix.runs-on || 'windows-2022' }} strategy: - matrix: - include: -# - platform: windows-2019 -# build_arch: x64 -# python_arch: x64 -# spec: 3.6 -# - platform: windows-2019 -# build_arch: x64 -# python_arch: x64 -# spec: 3.7 -# - platform: windows-2019 -# build_arch: x64 -# python_arch: x64 -# spec: 3.8 -# - platform: windows-2019 -# build_arch: x64 -# python_arch: x64 -# spec: 3.9 -# - platform: windows-2019 -# build_arch: x64 -# python_arch: x64 -# spec: '3.10' -# - platform: windows-2019 -# build_arch: x64 -# python_arch: x64 -# spec: '3.11' - - platform: windows-2019 - build_arch: x64 - python_arch: x64 - spec: '3.12.0-rc.1' -# - platform: windows-2019 -# build_arch: win32 -# python_arch: x86 -# spec: 3.6 -# - platform: windows-2019 -# build_arch: win32 -# python_arch: x86 -# spec: 3.7 -# - platform: windows-2019 -# build_arch: win32 -# python_arch: x86 -# spec: 3.8 -# - platform: windows-2019 -# build_arch: win32 -# python_arch: x86 -# spec: 3.9 -# - platform: windows-2019 -# build_arch: win32 -# python_arch: x86 -# spec: '3.10' -# - platform: windows-2019 -# build_arch: win32 -# python_arch: x86 -# spec: '3.11' - - platform: windows-2019 - build_arch: win32 - python_arch: x86 - spec: '3.12.0-rc.1' + fail-fast: false + matrix: ${{ fromJSON(needs.make_windows_pyyaml_matrix.outputs.matrix_json) }} steps: # autocrlf screws up tests under Windows - name: Set git to use LF @@ -423,48 +516,87 @@ jobs: git config --global core.autocrlf false git config --global core.eol lf - - name: Checkout pyyaml - uses: actions/checkout@v3 + - name: fetch sdist artifact + id: fetch_sdist + uses: actions/download-artifact@v4 + with: + name: ${{ needs.build_sdist.outputs.artifact_name }} - name: Get cached libyaml state id: cached_libyaml - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: libyaml - key: libyaml_${{matrix.platform}}_${{matrix.build_arch}}_${{env.LIBYAML_REF}} + key: libyaml_${{'windows'}}_${{ contains(matrix.spec, 'win_amd64') && 'x64' || 'win32' }}_${{env.LIBYAML_REF}} fail-on-cache-miss: true - - name: Install python ${{ matrix.spec }} - uses: actions/setup-python@v4 + - name: Install python + uses: actions/setup-python@v5 with: - architecture: ${{ matrix.python_arch }} - python-version: ${{ matrix.spec }} + python-version: 3.x - name: Build/Test/Package - env: - PYYAML_FORCE_CYTHON: 1 - PYYAML_FORCE_LIBYAML: 1 + id: build shell: bash + env: + CIBW_BUILD: ${{matrix.spec}} + CIBW_BUILD_VERBOSITY: 1 + CIBW_BEFORE_TEST: ls -l {package} + CIBW_TEST_COMMAND: pytest {package} + CIBW_TEST_REQUIRES: pytest + CIBW_PRERELEASE_PYTHONS: 1 + #CIBW_CONFIG_SETTINGS: | + # pyyaml_build_config='{"include_dirs": ["libyaml/include"], "library_dirs": ["libyaml/build/Release"], "define": [["YAML_DECLARE_STATIC", 1]], "force": 1}' run: | set -eux python -V - python -m pip install "Cython<3.0" setuptools wheel + python -m pip install -U --user ${{ matrix.cibw_version || 'cibuildwheel' }} + mkdir pyyaml + + tar zxf pyyaml*.tar.gz/pyyaml*.tar.gz --strip-components=1 -C pyyaml - python setup.py \ - --with-libyaml build_ext \ - -I libyaml/include \ - -L libyaml/build/Release \ - -D YAML_DECLARE_STATIC \ - build bdist_wheel + cat << 'EOF' > build_config.toml + [tool.cibuildwheel.config-settings] + pyyaml_build_config='{"force":1, "include_dirs": ["../libyaml/include"], "library_dirs": ["../libyaml/build/Release"], "define": [["YAML_DECLARE_STATIC", 1]], "force": 1}' + EOF - # run tests on built wheel - python -m pip install dist/*.whl - python tests/lib/test_all.py + python3 -m cibuildwheel --config-file $(pwd)/build_config.toml --platform auto --output-dir ./dist ./pyyaml + + echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT" + # FIXME: ensure exactly one artifact - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: dist + name: ${{ steps.build.outputs.artifact_name }} path: dist/*.whl if-no-files-found: error + if: ${{ env.skip_artifact_upload != 'true' }} + + merge_artifacts: + needs: [python_sdist, macos_pyyaml, linux_pyyaml, windows_pyyaml] + runs-on: ubuntu-22.04 + steps: + - name: merge all artifacts + uses: actions/upload-artifact/merge@v4 + with: + name: dist + delete-merged: true + if: ${{ env.skip_artifact_upload != 'true' }} + + + check: + if: always() + needs: + - python_sdist + - linux_pyyaml + - macos_pyyaml + - windows_pyyaml + - merge_artifacts + runs-on: ubuntu-latest + steps: + - name: Verify all previous jobs succeeded (provides a single check to sample for gating purposes) + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} ... diff --git a/.github/workflows/manual_artifact_build.yaml b/.github/workflows/manual_artifact_build.yaml deleted file mode 100644 index c669f95a..00000000 --- a/.github/workflows/manual_artifact_build.yaml +++ /dev/null @@ -1,469 +0,0 @@ ---- -# This is the actual artifact build/release workflow. This workflow exists temporarily -# because GHA doesn't support a dynamic/conditional matrix. Ensure changes are synced with ci.yaml. -name: Manual Artifact Build - -on: -# push: -# pull_request: -# types: [opened, synchronize, reopened] - workflow_dispatch: - -env: - LIBYAML_REPO: https://github.com/yaml/libyaml - LIBYAML_REF: 0.2.5 - -jobs: - python_sdist: - name: pyyaml sdist - runs-on: ubuntu-latest - steps: - - name: Checkout PyYAML - uses: actions/checkout@v3 - - - name: Install a python - uses: actions/setup-python@v4 - with: - python-version: 3.x - - - name: Build sdist - env: - PYYAML_FORCE_CYTHON: 1 - PYYAML_FORCE_LIBYAML: 0 - run: | - python -V - python -m pip install build - - python -m build . - - # Ensure exactly one artifact was produced. - [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || { - echo "Unexpected content in dist dir: '$(ls dist/*.tar.gz)'." - exit 1 - } - - - name: Test sdist - run: | - # Install some libyaml headers. - # TODO Should we smoke test the sdist against the libyaml we built? - sudo apt update - sudo apt install libyaml-dev -y - - # Ensure Cython is not present so we use only what's in the sdist. - python -m pip uninstall Cython -y || true - - # Pass no extra args. - # We should auto-install with libyaml since it's present. - python -m pip install dist/*.tar.gz -v - - python packaging/build/smoketest.py - - - name: Upload sdist artifact - uses: actions/upload-artifact@v3 - with: - name: dist - path: dist/*.tar.gz - - - linux_libyaml: - name: libyaml ${{matrix.cfg.arch}} ${{matrix.cfg.platform}} - runs-on: ubuntu-latest - strategy: - matrix: - cfg: - - { platform: manylinux1, arch: x86_64 } - - { platform: manylinux2014, arch: x86_64 } - - { platform: manylinux2014, arch: aarch64 } - - { platform: manylinux2014, arch: s390x } - - { platform: musllinux_1_1, arch: x86_64 } - - env: - DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}} - steps: - - name: Check cached libyaml state - id: cached_libyaml - uses: actions/cache@v3 - with: - path: libyaml - key: libyaml_${{matrix.cfg.platform}}_${{matrix.cfg.arch}}_${{env.LIBYAML_REF}} - - - name: configure docker foreign arch support - uses: docker/setup-qemu-action@v2 - if: matrix.cfg.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true' - - - name: Checkout pyyaml - uses: actions/checkout@v3 - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - - name: Build libyaml - run: > - docker run --rm - --volume "$(pwd):/io" - --env LIBYAML_REF - --env LIBYAML_REPO - --workdir /io - "$DOCKER_IMAGE" - /io/packaging/build/libyaml.sh - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - - name: ensure output is world readable (or cache fill fails with Permission Denied) - run: > - sudo chmod -R a+r ./libyaml/ - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - - linux_pyyaml: - needs: linux_libyaml - name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.spec}} - runs-on: ubuntu-latest - strategy: - matrix: - include: - - { platform: manylinux1, arch: x86_64, spec: cp36 } - - { platform: manylinux1, arch: x86_64, spec: cp37 } - - { platform: manylinux1, arch: x86_64, spec: cp38 } - - { platform: manylinux1, arch: x86_64, spec: cp39 } - - { platform: manylinux2014, arch: x86_64, spec: cp310 } - - { platform: manylinux2014, arch: x86_64, spec: cp311 } - - { platform: manylinux2014, arch: x86_64, spec: cp312 } - - { platform: manylinux2014, arch: aarch64, spec: cp36 } - - { platform: manylinux2014, arch: aarch64, spec: cp37 } - - { platform: manylinux2014, arch: aarch64, spec: cp38 } - - { platform: manylinux2014, arch: aarch64, spec: cp39 } - - { platform: manylinux2014, arch: aarch64, spec: cp310 } - - { platform: manylinux2014, arch: aarch64, spec: cp311 } - - { platform: manylinux2014, arch: aarch64, spec: cp312 } - - { platform: manylinux2014, arch: s390x, spec: cp36 } - - { platform: manylinux2014, arch: s390x, spec: cp37 } - - { platform: manylinux2014, arch: s390x, spec: cp38 } - - { platform: manylinux2014, arch: s390x, spec: cp39 } - - { platform: manylinux2014, arch: s390x, spec: cp310 } - - { platform: manylinux2014, arch: s390x, spec: cp311 } - - { platform: manylinux2014, arch: s390x, spec: cp312 } - - { platform: musllinux_1_1, arch: x86_64, spec: cp38 } - - { platform: musllinux_1_1, arch: x86_64, spec: cp39 } - - { platform: musllinux_1_1, arch: x86_64, spec: cp310 } - - { platform: musllinux_1_1, arch: x86_64, spec: cp311 } - - { platform: musllinux_1_1, arch: x86_64, spec: cp312 } - - steps: - - name: Checkout PyYAML - uses: actions/checkout@v3 - - - name: Fetch cached libyaml - id: cached_libyaml - uses: actions/cache/restore@v3 - with: - path: libyaml - key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - fail-on-cache-miss: true - - - name: configure docker foreign arch support - uses: docker/setup-qemu-action@v2 - if: matrix.arch != 'x86_64' - - - name: Build/Test/Package - env: - CIBW_ARCHS: all - # HACK: ick, maybe deconstruct the matrix a bit or query cibuildwheel for its default target *linux spec first? - CIBW_BUILD: ${{matrix.spec}}-${{ contains(matrix.platform, 'musllinux') && 'musllinux' || 'manylinux' }}_${{matrix.arch}} - CIBW_BUILD_VERBOSITY: 1 - # containerized Linux builds require explicit CIBW_ENVIRONMENT - CIBW_ENVIRONMENT: > - C_INCLUDE_PATH=libyaml/include - LIBRARY_PATH=libyaml/src/.libs - LD_LIBRARY_PATH=libyaml/src/.libs - PYYAML_FORCE_CYTHON=1 - PYYAML_FORCE_LIBYAML=1 - CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py - run: | - set -eux - python3 -V - python3 -m pip install -U --user cibuildwheel - python3 -m cibuildwheel --platform auto --output-dir dist . - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: dist - path: dist/*.whl - if-no-files-found: error - - macos_libyaml: - name: libyaml macos ${{matrix.arch}} - strategy: - matrix: - include: - - arch: x86_64 - - arch: arm64 - runs_on: [self-hosted, macOS, arm64] - deployment_target: '11.0' - run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} - sdkroot: macosx11.3 - defaults: - run: - shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }} - runs-on: ${{ matrix.runs_on || 'macos-11' }} - steps: - - name: Check cached libyaml state - id: cached_libyaml - uses: actions/cache@v3 - with: - path: libyaml - key: libyaml_macos_${{matrix.arch}}_${{env.LIBYAML_REF}} - - - name: Checkout PyYAML - uses: actions/checkout@v3 - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - - name: Build libyaml - env: - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }} - SDKROOT: ${{ matrix.sdkroot || 'macosx' }} - run: | - set -eux - brew install automake coreutils m4 - bash ./packaging/build/libyaml.sh - echo "finished artifact arch is $(lipo -archs libyaml/src/.libs/libyaml.a)" - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - - macos_pyyaml: - needs: macos_libyaml - name: pyyaml ${{ matrix.spec }} - runs-on: ${{ matrix.runs_on || 'macos-11' }} - defaults: - run: - shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }} - strategy: - matrix: - include: - - spec: cp36-macosx_x86_64 - cibw_version: cibuildwheel==2.11.1 - - spec: cp37-macosx_x86_64 - cibw_version: cibuildwheel==2.11.1 - - spec: cp38-macosx_x86_64 - cibw_version: cibuildwheel==2.11.1 - - spec: cp39-macosx_x86_64 - - spec: cp310-macosx_x86_64 - - spec: cp311-macosx_x86_64 - - spec: cp312-macosx_x86_64 - - # build for arm64 under a hacked macOS 12 self-hosted x86_64-on-arm64 runner until arm64 is fully supported - - spec: cp39-macosx_arm64 - deployment_target: '11.0' - runs_on: [self-hosted, macOS, arm64] - arch: arm64 - run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} - sdkroot: macosx11.3 - - - spec: cp310-macosx_arm64 - deployment_target: '11.0' - runs_on: [self-hosted, macOS, arm64] - arch: arm64 - run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} - sdkroot: macosx11.3 - - - spec: cp311-macosx_arm64 - deployment_target: '11.0' - runs_on: [self-hosted, macOS, arm64] - arch: arm64 - run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} - sdkroot: macosx11.3 - - - spec: cp312-macosx_arm64 - deployment_target: '11.0' - runs_on: [self-hosted, macOS, arm64] - arch: arm64 - run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} - sdkroot: macosx11.3 - - steps: - - name: Checkout PyYAML - uses: actions/checkout@v3 - - - name: Get cached libyaml state - id: cached_libyaml - uses: actions/cache/restore@v3 - with: - path: libyaml - key: libyaml_macos_${{ matrix.arch || 'x86_64' }}_${{env.LIBYAML_REF}} - fail-on-cache-miss: true - - - name: Build/Test/Package - env: - C_INCLUDE_PATH: libyaml/include - CIBW_BUILD: ${{matrix.spec}} - CIBW_BUILD_VERBOSITY: 1 - CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py - LIBRARY_PATH: libyaml/src/.libs - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }} - SDKROOT: ${{ matrix.sdkroot || 'macosx' }} - run: | - python3 -V - python3 -m pip install -U --user ${{ matrix.cibw_version || 'cibuildwheel' }} - python3 -m cibuildwheel --platform auto --output-dir dist . - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: dist - path: dist/*.whl - if-no-files-found: error - - windows_libyaml: - name: libyaml ${{matrix.platform}} ${{matrix.arch}} - runs-on: ${{matrix.platform}} - strategy: - matrix: - include: - - platform: windows-2019 - arch: x64 - - platform: windows-2019 - arch: win32 - steps: - - name: Get cached libyaml state - id: cached_libyaml - uses: actions/cache@v3 - with: - path: libyaml - key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - - - name: Build libyaml - shell: bash - if: steps.cached_libyaml.outputs.cache-hit != 'true' - run: | - # git spews all over stderr unless we tell it not to - export GIT_REDIRECT_STDERR="2>&1" - - if [[ ! -d ./libyaml ]]; then - git clone -b ${{ env.LIBYAML_REF }} ${{ env.LIBYAML_REPO }} 2>&1 - fi - - pushd libyaml - git clean -fdx - popd - - mkdir libyaml/build - - pushd libyaml/build - cmake.exe -G "Visual Studio 16 2019" -A ${{ matrix.arch }} -DYAML_STATIC_LIB_NAME=yaml .. - cmake.exe --build . --config Release - popd - - - windows_pyyaml: - needs: windows_libyaml - name: pyyaml ${{ matrix.platform }} ${{matrix.python_arch}} ${{matrix.spec}} - runs-on: ${{matrix.platform}} - strategy: - matrix: - include: - - platform: windows-2019 - build_arch: x64 - python_arch: x64 - spec: 3.6 - - platform: windows-2019 - build_arch: x64 - python_arch: x64 - spec: 3.7 - - platform: windows-2019 - build_arch: x64 - python_arch: x64 - spec: 3.8 - - platform: windows-2019 - build_arch: x64 - python_arch: x64 - spec: 3.9 - - platform: windows-2019 - build_arch: x64 - python_arch: x64 - spec: '3.10' - - platform: windows-2019 - build_arch: x64 - python_arch: x64 - spec: '3.11' - - platform: windows-2019 - build_arch: x64 - python_arch: x64 - spec: '3.12.0-rc.1' - - platform: windows-2019 - build_arch: win32 - python_arch: x86 - spec: 3.6 - - platform: windows-2019 - build_arch: win32 - python_arch: x86 - spec: 3.7 - - platform: windows-2019 - build_arch: win32 - python_arch: x86 - spec: 3.8 - - platform: windows-2019 - build_arch: win32 - python_arch: x86 - spec: 3.9 - - platform: windows-2019 - build_arch: win32 - python_arch: x86 - spec: '3.10' - - platform: windows-2019 - build_arch: win32 - python_arch: x86 - spec: '3.11' - - platform: windows-2019 - build_arch: win32 - python_arch: x86 - spec: '3.12.0-rc.1' - steps: - # autocrlf screws up tests under Windows - - name: Set git to use LF - run: | - git config --global core.autocrlf false - git config --global core.eol lf - - - name: Checkout pyyaml - uses: actions/checkout@v3 - - - name: Get cached libyaml state - id: cached_libyaml - uses: actions/cache/restore@v3 - with: - path: libyaml - key: libyaml_${{matrix.platform}}_${{matrix.build_arch}}_${{env.LIBYAML_REF}} - fail-on-cache-miss: true - - - name: Install python ${{ matrix.spec }} - uses: actions/setup-python@v4 - with: - architecture: ${{ matrix.python_arch }} - python-version: ${{ matrix.spec }} - - - name: Build/Test/Package - env: - PYYAML_FORCE_CYTHON: 1 - PYYAML_FORCE_LIBYAML: 1 - shell: bash - run: | - set -eux - python -V - python -m pip install "Cython<3.0" setuptools wheel - - python setup.py \ - --with-libyaml build_ext \ - -I libyaml/include \ - -L libyaml/build/Release \ - -D YAML_DECLARE_STATIC \ - build bdist_wheel - - # run tests on built wheel - python -m pip install dist/*.whl - python tests/lib/test_all.py - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: dist - path: dist/*.whl - if-no-files-found: error -... diff --git a/MANIFEST.in b/MANIFEST.in index 3ab0c4f4..d8d73aa9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ include CHANGES README LICENSE Makefile pyproject.toml setup.py recursive-include lib/yaml *.py recursive-include lib/_yaml *.py +recursive-include packaging * recursive-include examples *.py *.cfg *.yaml recursive-include tests/data * recursive-include tests/lib *.py diff --git a/packaging/_pyyaml_pep517.py b/packaging/_pyyaml_pep517.py new file mode 100644 index 00000000..5f390266 --- /dev/null +++ b/packaging/_pyyaml_pep517.py @@ -0,0 +1,51 @@ +import inspect + + +def _bridge_build_meta(): + import functools + import sys + + from setuptools import build_meta + + self_module = sys.modules[__name__] + + for attr_name in build_meta.__all__: + attr_value = getattr(build_meta, attr_name) + if callable(attr_value): + setattr(self_module, attr_name, functools.partial(_expose_config_settings, attr_value)) + + +class ActiveConfigSettings: + _current = {} + + def __init__(self, config_settings): + self._config = config_settings + + def __enter__(self): + type(self)._current = self._config + + def __exit__(self, exc_type, exc_val, exc_tb): + type(self)._current = {} + + @classmethod + def current(cls): + return cls._current + + +def _expose_config_settings(real_method, *args, **kwargs): + from contextlib import nullcontext + import inspect + + sig = inspect.signature(real_method) + boundargs = sig.bind(*args, **kwargs) + + config = boundargs.arguments.get('config_settings') + + ctx = ActiveConfigSettings(config) if config else nullcontext() + + with ctx: + return real_method(*args, **kwargs) + + +_bridge_build_meta() + diff --git a/pyproject.toml b/pyproject.toml index 4bc04c0d..d8e5b969 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,9 @@ [build-system] -requires = ["setuptools", "wheel", "Cython<3.0"] -build-backend = "setuptools.build_meta" +requires = [ + "setuptools", # FIXME: declare min/max setuptools versions? + "wheel", + "Cython; python_version < '3.13'", + "Cython>=3.0; python_version >= '3.13'" +] +backend-path = ["packaging"] +build-backend = "_pyyaml_pep517" diff --git a/setup.py b/setup.py index 65b0ea0e..cf692826 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,8 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries :: Python Modules", @@ -82,7 +84,12 @@ with_cython = True try: from Cython.Distutils.extension import Extension as _Extension - from Cython.Distutils import build_ext as _build_ext + try: + # try old_build_ext from Cython > 3 first, until we can dump it entirely + from Cython.Distutils.old_build_ext import old_build_ext as _build_ext + except ImportError: + # Cython < 3 + from Cython.Distutils import build_ext as _build_ext with_cython = True except ImportError: if with_cython: @@ -94,6 +101,14 @@ bdist_wheel = None +try: + from _pyyaml_pep517 import ActiveConfigSettings +except ImportError: + class ActiveConfigSettings: + @staticmethod + def current(): + return {} + # on Windows, disable wheel generation warning noise windows_ignore_warnings = [ "Unknown distribution option: 'python_requires'", @@ -173,6 +188,31 @@ def __init__(self, name, sources, feature_name, feature_description, class build_ext(_build_ext): + def finalize_options(self): + super().finalize_options() + pep517_config = ActiveConfigSettings.current() + + build_config = pep517_config.get('pyyaml_build_config') + + if build_config: + import json + build_config = json.loads(build_config) + print(f"`pyyaml_build_config`: {build_config}") + else: + build_config = {} + print("No `pyyaml_build_config` setting found.") + + for key, value in build_config.items(): + existing_value = getattr(self, key, ...) + if existing_value is ...: + print(f"ignoring unknown config key {key!r}") + continue + + if existing_value: + print(f"combining {key!r} {existing_value!r} and {value!r}") + value = existing_value + value # FIXME: handle type diff + + setattr(self, key, value) def run(self): optional = True @@ -230,6 +270,7 @@ def build_extensions(self): if with_ext is not None and not with_ext: continue if with_cython: + print(f"BUILDING CYTHON EXT; {self.include_dirs=} {self.library_dirs=} {self.define=}") ext.sources = self.cython_sources(ext.sources, ext) try: self.build_extension(ext) diff --git a/tests/lib/canonical.py b/tests/legacy_tests/canonical.py similarity index 100% rename from tests/lib/canonical.py rename to tests/legacy_tests/canonical.py diff --git a/tests/legacy_tests/conftest.py b/tests/legacy_tests/conftest.py new file mode 100644 index 00000000..e9e813ba --- /dev/null +++ b/tests/legacy_tests/conftest.py @@ -0,0 +1,132 @@ +# pytest custom collection adapter for legacy pyyaml unit tests/data files; surfaces each +# legacy test case as a pyyaml item + +import os +import pathlib +import pytest +import warnings + +from test_appliance import find_test_filenames, DATA + +try: + from yaml import _yaml + HAS_LIBYAML_EXT = True + del _yaml +except ImportError: + HAS_LIBYAML_EXT = False + + +_test_filenames = find_test_filenames(DATA) + +# ignore all datafiles +collect_ignore_glob = ['data/*'] + + +class PyYAMLItem(pytest.Item): + def __init__(self, parent=None, config=None, session=None, nodeid=None, function=None, filenames=None, **kwargs): + self._function = function + self._fargs = filenames or [] + + super().__init__(os.path.basename(filenames[0]) if filenames else parent.name, parent, config, session, nodeid) + # this is gnarly since the type of fspath is private; fixed in pytest 7 to use pathlib on the `path` attr + if filenames: # pass the data file location as the test path + self.fspath = parent.fspath.__class__(filenames[0]) + self.lineno = 1 + else: # pass the function location in the code + self.fspath = parent.fspath.__class__(function.__code__.co_filename) + self.lineno = function.__code__.co_firstlineno + + def runtest(self): + self._function(verbose=True, *self._fargs) + + def reportinfo(self): + return self.fspath, self.lineno, '' + + +class PyYAMLCollector(pytest.Collector): + def __init__(self, name, parent=None, function=None, **kwargs): + self._function = function + self.fspath = parent.fspath.__class__(function.__code__.co_filename) + self.lineno = function.__code__.co_firstlineno + + # avoid fspath deprecation warnings on pytest < 7 + if hasattr(self, 'path') and 'fspath' in kwargs: + del kwargs['fspath'] + + super().__init__(name=name, parent=parent, **kwargs) + + def collect(self): + items = [] + + unittest = getattr(self._function, 'unittest', None) + + if unittest is True: # no filenames + items.append(PyYAMLItem.from_parent(parent=self, function=self._function, filenames=None)) + else: + for base, exts in _test_filenames: + filenames = [] + for ext in unittest: + if ext not in exts: + break + filenames.append(os.path.join(DATA, base + ext)) + else: + skip_exts = getattr(self._function, 'skip', []) + for skip_ext in skip_exts: + if skip_ext in exts: + break + else: + items.append(PyYAMLItem.from_parent(parent=self, function=self._function, filenames=filenames)) + + return items or None + + def reportinfo(self): + return self.fspath, self.lineno, '' + + @classmethod + def from_parent(cls, parent, fspath, **kwargs): + return super().from_parent(parent=parent, fspath=fspath, **kwargs) + + +@pytest.hookimpl(hookwrapper=True, trylast=True) +def pytest_pycollect_makeitem(collector, name: str, obj: object): + outcome = yield + outcome.get_result() + if not callable(obj): + outcome.force_result(None) + return + unittest = getattr(obj, 'unittest', None) + + if not unittest: + outcome.force_result(None) + return + + if unittest is True: # no file list to run against, just return a test item instead of a collector + outcome.force_result(PyYAMLItem.from_parent(name=name, parent=collector, fspath=collector.fspath, function=obj)) + return + + # there's a file list; return a collector to create individual items for each + outcome.force_result(PyYAMLCollector.from_parent(name=name, parent=collector, fspath=collector.fspath, function=obj)) + return + + +def pytest_collection_modifyitems(session, config, items): + pass + + +def pytest_ignore_collect(collection_path: pathlib.Path): + basename = collection_path.name + # ignore all Python files in this subtree for normal pytest collection + if basename not in ['test_yaml.py', 'test_yaml_ext.py']: + return True + + # ignore extension tests (depending on config) + if basename == 'test_yaml_ext.py': + require_libyaml = os.environ.get('PYYAML_FORCE_LIBYAML', None) + if require_libyaml == '1' and not HAS_LIBYAML_EXT: + raise RuntimeError('PYYAML_FORCE_LIBYAML envvar is set, but libyaml extension is not available') + if require_libyaml == '0': + return True + if not HAS_LIBYAML_EXT: + warnings.warn('libyaml extension is not available, skipping libyaml tests') + return True + diff --git a/tests/lib/test_all.py b/tests/legacy_tests/test_all.py similarity index 100% rename from tests/lib/test_all.py rename to tests/legacy_tests/test_all.py diff --git a/tests/lib/test_appliance.py b/tests/legacy_tests/test_appliance.py similarity index 100% rename from tests/lib/test_appliance.py rename to tests/legacy_tests/test_appliance.py diff --git a/tests/lib/test_build.py b/tests/legacy_tests/test_build.py similarity index 100% rename from tests/lib/test_build.py rename to tests/legacy_tests/test_build.py diff --git a/tests/lib/test_build_ext.py b/tests/legacy_tests/test_build_ext.py similarity index 100% rename from tests/lib/test_build_ext.py rename to tests/legacy_tests/test_build_ext.py diff --git a/tests/lib/test_canonical.py b/tests/legacy_tests/test_canonical.py similarity index 100% rename from tests/lib/test_canonical.py rename to tests/legacy_tests/test_canonical.py diff --git a/tests/lib/test_constructor.py b/tests/legacy_tests/test_constructor.py similarity index 100% rename from tests/lib/test_constructor.py rename to tests/legacy_tests/test_constructor.py diff --git a/tests/lib/test_dump_load.py b/tests/legacy_tests/test_dump_load.py similarity index 100% rename from tests/lib/test_dump_load.py rename to tests/legacy_tests/test_dump_load.py diff --git a/tests/lib/test_emitter.py b/tests/legacy_tests/test_emitter.py similarity index 100% rename from tests/lib/test_emitter.py rename to tests/legacy_tests/test_emitter.py diff --git a/tests/lib/test_errors.py b/tests/legacy_tests/test_errors.py similarity index 100% rename from tests/lib/test_errors.py rename to tests/legacy_tests/test_errors.py diff --git a/tests/lib/test_input_output.py b/tests/legacy_tests/test_input_output.py similarity index 100% rename from tests/lib/test_input_output.py rename to tests/legacy_tests/test_input_output.py diff --git a/tests/lib/test_mark.py b/tests/legacy_tests/test_mark.py similarity index 100% rename from tests/lib/test_mark.py rename to tests/legacy_tests/test_mark.py diff --git a/tests/lib/test_multi_constructor.py b/tests/legacy_tests/test_multi_constructor.py similarity index 100% rename from tests/lib/test_multi_constructor.py rename to tests/legacy_tests/test_multi_constructor.py diff --git a/tests/lib/test_reader.py b/tests/legacy_tests/test_reader.py similarity index 100% rename from tests/lib/test_reader.py rename to tests/legacy_tests/test_reader.py diff --git a/tests/lib/test_recursive.py b/tests/legacy_tests/test_recursive.py similarity index 100% rename from tests/lib/test_recursive.py rename to tests/legacy_tests/test_recursive.py diff --git a/tests/lib/test_representer.py b/tests/legacy_tests/test_representer.py similarity index 100% rename from tests/lib/test_representer.py rename to tests/legacy_tests/test_representer.py diff --git a/tests/lib/test_resolver.py b/tests/legacy_tests/test_resolver.py similarity index 100% rename from tests/lib/test_resolver.py rename to tests/legacy_tests/test_resolver.py diff --git a/tests/lib/test_schema.py b/tests/legacy_tests/test_schema.py similarity index 100% rename from tests/lib/test_schema.py rename to tests/legacy_tests/test_schema.py diff --git a/tests/lib/test_sort_keys.py b/tests/legacy_tests/test_sort_keys.py similarity index 100% rename from tests/lib/test_sort_keys.py rename to tests/legacy_tests/test_sort_keys.py diff --git a/tests/lib/test_structure.py b/tests/legacy_tests/test_structure.py similarity index 100% rename from tests/lib/test_structure.py rename to tests/legacy_tests/test_structure.py diff --git a/tests/lib/test_tokens.py b/tests/legacy_tests/test_tokens.py similarity index 100% rename from tests/lib/test_tokens.py rename to tests/legacy_tests/test_tokens.py diff --git a/tests/lib/test_yaml.py b/tests/legacy_tests/test_yaml.py similarity index 100% rename from tests/lib/test_yaml.py rename to tests/legacy_tests/test_yaml.py diff --git a/tests/lib/test_yaml_ext.py b/tests/legacy_tests/test_yaml_ext.py similarity index 100% rename from tests/lib/test_yaml_ext.py rename to tests/legacy_tests/test_yaml_ext.py