diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d517cf83..61d42b2e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,6 +7,12 @@ on: # types: [opened, synchronize, reopened] workflow_dispatch: inputs: + libyaml_repo: + type: string + default: https://github.com/yaml/libyaml + libyaml_ref: + type: string + default: 0.2.5 skip_artifact_upload: type: boolean default: true @@ -18,206 +24,226 @@ on: default: true env: - LIBYAML_REPO: https://github.com/yaml/libyaml - LIBYAML_REF: 0.2.5 + LIBYAML_REPO: ${{ inputs.libyaml_repo || 'https://github.com/yaml/libyaml' }} # FIXME: can we ref the input.default value? + LIBYAML_REF: ${{ inputs.libyaml_ref || '0.2.5' }} # FIXME: can we ref the input.default value? skip_ci_redundant_jobs: ${{ inputs.skip_ci_redundant_jobs || github.event_name == 'pull_request' || github.event_name == 'push' }} skip_slow_jobs: ${{ inputs.skip_slow_jobs || github.event_name == 'pull_request' || github.event_name == 'push' }} skip_artifact_upload: ${{ inputs.skip_artifact_upload || true }} # ${{ github.event_name == 'pull_request' }} jobs: -# python_sdist: -# name: pyyaml sdist -# runs-on: ubuntu-22.04 -# steps: -# - name: Checkout PyYAML -# uses: actions/checkout@v4 -# -# - name: Install a python -# uses: actions/setup-python@v5 -# with: -# python-version: 3.x -# -# - name: Build sdist -# env: -# PYYAML_FORCE_CYTHON: 1 -# PYYAML_FORCE_LIBYAML: 0 -# run: | -# python -V -# python -m pip install build -# -# python -m build . -# -# # Ensure exactly one artifact was produced. -# [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || { -# echo "Unexpected content in dist dir: '$(ls dist/*.tar.gz)'." -# exit 1 -# } -# -# - name: Test sdist -# run: | -# # Install some libyaml headers. -# # TODO Should we smoke test the sdist against the libyaml we built? -# sudo apt update -# sudo apt install libyaml-dev -y -# -# # Ensure Cython is not present so we use only what's in the sdist. -# python -m pip uninstall Cython -y || true -# -# # Pass no extra args. -# # We should auto-install with libyaml since it's present. -# python -m pip install dist/*.tar.gz -v -# -# python packaging/build/smoketest.py -# -# - name: Upload sdist artifact -# uses: actions/upload-artifact@v3 -# with: -# name: dist -# path: dist/*.tar.gz -# -# -# linux_libyaml: -# name: libyaml ${{matrix.cfg.arch}} ${{matrix.cfg.platform}} -# runs-on: ubuntu-22.04 -# strategy: -# # FIXME: dynamic matrix away the slow ones -# matrix: -# cfg: -# - { platform: manylinux1, arch: x86_64 } -# - { platform: manylinux2014, arch: x86_64 } -# - { platform: manylinux2014, arch: aarch64 } -# - { platform: manylinux2014, arch: s390x } -# - { platform: musllinux_1_1, arch: x86_64 } -# - { platform: musllinux_1_1, arch: aarch64 } -# env: -# DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}} -# steps: -# - name: Check cached libyaml state -# id: cached_libyaml -# uses: actions/cache@v4 -# with: -# path: libyaml -# key: libyaml_${{matrix.cfg.platform}}_${{matrix.cfg.arch}}_${{env.LIBYAML_REF}} -# -# - name: configure docker foreign arch support -# uses: docker/setup-qemu-action@v3 -# if: matrix.cfg.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true' -# -# - name: Checkout pyyaml -# uses: actions/checkout@v4 -# if: steps.cached_libyaml.outputs.cache-hit != 'true' -# -# - name: Build libyaml -# run: > -# docker run --rm -# --volume "$(pwd):/io" -# --env LIBYAML_REF -# --env LIBYAML_REPO -# --workdir /io -# "$DOCKER_IMAGE" -# /io/packaging/build/libyaml.sh -# if: steps.cached_libyaml.outputs.cache-hit != 'true' -# -# - name: ensure output is world readable (or cache fill fails with Permission Denied) -# run: > -# sudo chmod -R a+r ./libyaml/ -# if: steps.cached_libyaml.outputs.cache-hit != 'true' -# -# make_linux_pyyaml_matrix: -# runs-on: ubuntu-22.04 -# outputs: -# matrix_json: ${{ steps.make_matrix.outputs.matrix_json }} -# steps: -# - uses: actions/checkout@v4 -# - name: make a matrix -# id: make_matrix -# uses: ./.github/actions/dynamatrix -# with: -# matrix_yaml: | -# include: -# - { platform: manylinux1, arch: x86_64, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs }} } -# - { platform: manylinux1, arch: x86_64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs }} } -# - { platform: manylinux2014, arch: x86_64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs }} } -# - { platform: manylinux2014, arch: x86_64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs }} } -# - { platform: manylinux2014, arch: x86_64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs }} } -# - { platform: manylinux2014, arch: x86_64, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip" } -# - { platform: manylinux2014, arch: aarch64, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: manylinux2014, arch: aarch64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: manylinux2014, arch: aarch64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: manylinux2014, arch: aarch64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: manylinux2014, arch: aarch64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: manylinux2014, arch: aarch64, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip", omit: ${{ env.skip_slow_jobs }} } -# - { platform: manylinux2014, arch: s390x, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: manylinux2014, arch: s390x, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: manylinux2014, arch: s390x, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: manylinux2014, arch: s390x, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: manylinux2014, arch: s390x, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: manylinux2014, arch: s390x, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip", omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: musllinux_1_1, arch: x86_64, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs }} } -# - { platform: musllinux_1_1, arch: x86_64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs }} } -# - { platform: musllinux_1_1, arch: x86_64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs }} } -# - { platform: musllinux_1_1, arch: x86_64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs }} } -# - { platform: musllinux_1_1, arch: x86_64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs }} } -# - { platform: musllinux_1_1, arch: x86_64, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip", omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: musllinux_1_1, arch: aarch64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: musllinux_1_1, arch: aarch64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: musllinux_1_1, arch: aarch64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: musllinux_1_1, arch: aarch64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } -# - { platform: musllinux_1_1, arch: aarch64, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip", omit: ${{ env.skip_slow_jobs }} } -# -# -# linux_pyyaml: -# needs: [linux_libyaml, make_linux_pyyaml_matrix] -# name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.spec}} -# runs-on: ubuntu-22.04 -# strategy: -# fail-fast: false -# matrix: ${{ fromJSON(needs.make_linux_pyyaml_matrix.outputs.matrix_json) }} -# -# steps: -# - name: Checkout PyYAML -# uses: actions/checkout@v4 -# -# - name: Fetch cached libyaml -# id: cached_libyaml -# uses: actions/cache/restore@v4 -# with: -# path: libyaml -# key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} -# fail-on-cache-miss: true -# -# - name: configure docker foreign arch support -# uses: docker/setup-qemu-action@v3 -# if: matrix.arch != 'x86_64' -# -# - name: Build/Test/Package -# env: -# CIBW_ARCHS: all -# # HACK: ick, maybe deconstruct the matrix a bit or query cibuildwheel for its default target *linux spec first? -# CIBW_BUILD: ${{matrix.spec}}-${{ contains(matrix.platform, 'musllinux') && 'musllinux' || 'manylinux' }}_${{matrix.arch}} -# CIBW_BUILD_VERBOSITY: 1 -# # containerized Linux builds require explicit CIBW_ENVIRONMENT -# CIBW_ENVIRONMENT: > -# C_INCLUDE_PATH=libyaml/include -# LIBRARY_PATH=libyaml/src/.libs -# LD_LIBRARY_PATH=libyaml/src/.libs -# PYYAML_FORCE_CYTHON=1 -# PYYAML_FORCE_LIBYAML=1 -# CIBW_TEST_COMMAND: cd {project}; pytest -# CIBW_TEST_REQUIRES: pytest -# run: | -# set -eux -# python3 -V -# python3 -m pip install -U --user ${{ matrix.cibw_version || 'cibuildwheel' }} -# python3 -m cibuildwheel --platform auto --output-dir dist . -# -# - name: Upload artifacts -# uses: actions/upload-artifact@v3 -# with: -# name: dist -# path: dist/*.whl -# if-no-files-found: error -# if: ${{ ! env.skip_artifact_upload }} + python_sdist: + name: pyyaml sdist + runs-on: ubuntu-22.04 + steps: + - name: Checkout PyYAML + uses: actions/checkout@v4 + + - name: Install a python + uses: actions/setup-python@v5 + with: + python-version: 3.x + + - name: Build sdist + env: + PYYAML_FORCE_CYTHON: 1 + PYYAML_FORCE_LIBYAML: 0 + run: | + python -V + python -m pip install build + + python -m build . + + # Ensure exactly one artifact was produced. + [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || { + echo "Unexpected content in dist dir: '$(ls dist/*.tar.gz)'." + exit 1 + } + + - name: Test sdist + run: | + # Install some libyaml headers. + # TODO Should we smoke test the sdist against the libyaml we built? + sudo apt update + sudo apt install libyaml-dev -y + + # Ensure Cython is not present so we use only what's in the sdist. + python -m pip uninstall Cython -y || true + + # Pass no extra args. + # We should auto-install with libyaml since it's present. + python -m pip install dist/*.tar.gz -v + + python packaging/build/smoketest.py + + echo "ARTIFACT_NAME=$(ls ./dist/)" >> "$GITHUB_OUTPUT" + # FIXME: ensure exactly one artifact + + - name: Upload sdist artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.build.outputs.ARTIFACT_NAME }} + path: dist/${{ steps.build.outputs.ARTIFACT_NAME }} + if-no-files-found: error + if: ${{ ! env.skip_artifact_upload }} + + make_linux_libyaml_matrix: + runs-on: ubuntu-22.04 + outputs: + matrix_json: ${{ steps.make_matrix.outputs.matrix_json }} + steps: + - uses: actions/checkout@v4 + - name: make a matrix + id: make_matrix + uses: ./.github/actions/dynamatrix + with: + matrix_yaml: | + include: + - { platform: manylinux1, arch: x86_64 } + - { platform: manylinux2014, arch: x86_64 } + - { platform: manylinux2014, arch: aarch64, omit: ${{ env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, omit: ${{ env.skip_slow_jobs }} } + - { platform: musllinux_1_1, arch: x86_64 } + - { platform: musllinux_1_1, arch: aarch64 } + + linux_libyaml: + name: libyaml ${{matrix.arch}} ${{matrix.platform}} + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.make_linux_libyaml_matrix.outputs.matrix_json) }} + env: + DOCKER_IMAGE: quay.io/pypa/${{matrix.platform}}_${{matrix.arch}} + steps: + - name: Check cached libyaml state + id: cached_libyaml + uses: actions/cache@v4 + with: + path: libyaml + key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} + + - name: configure docker foreign arch support + uses: docker/setup-qemu-action@v3 + if: matrix.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: Checkout pyyaml + uses: actions/checkout@v4 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: Build libyaml + run: > + docker run --rm + --volume "$(pwd):/io" + --env LIBYAML_REF + --env LIBYAML_REPO + --workdir /io + "$DOCKER_IMAGE" + /io/packaging/build/libyaml.sh + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: ensure output is world readable (or cache fill fails with Permission Denied) + run: > + sudo chmod -R a+r ./libyaml/ + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + make_linux_pyyaml_matrix: + runs-on: ubuntu-22.04 + outputs: + matrix_json: ${{ steps.make_matrix.outputs.matrix_json }} + steps: + - uses: actions/checkout@v4 + - name: make a matrix + id: make_matrix + uses: ./.github/actions/dynamatrix + with: + matrix_yaml: | + include: + - { platform: manylinux1, arch: x86_64, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: manylinux1, arch: x86_64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: manylinux2014, arch: x86_64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: manylinux2014, arch: x86_64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: manylinux2014, arch: x86_64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: manylinux2014, arch: x86_64, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip" } + - { platform: manylinux2014, arch: aarch64, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: aarch64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: aarch64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: aarch64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: aarch64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: aarch64, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip", omit: ${{ env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: manylinux2014, arch: s390x, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip", omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: musllinux_1_1, arch: x86_64, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: musllinux_1_1, arch: x86_64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: musllinux_1_1, arch: x86_64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: musllinux_1_1, arch: x86_64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: musllinux_1_1, arch: x86_64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs }} } + - { platform: musllinux_1_1, arch: x86_64, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip", omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: musllinux_1_1, arch: aarch64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: musllinux_1_1, arch: aarch64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: musllinux_1_1, arch: aarch64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: musllinux_1_1, arch: aarch64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} } + - { platform: musllinux_1_1, arch: aarch64, spec: cp313, cibw_version: "https://github.com/nitzmahone/cibuildwheel/archive/refs/heads/py313_support.zip", omit: ${{ env.skip_slow_jobs }} } + + + linux_pyyaml: + needs: [linux_libyaml, make_linux_pyyaml_matrix] + name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.spec}} + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.make_linux_pyyaml_matrix.outputs.matrix_json) }} + + steps: + - name: Checkout PyYAML + uses: actions/checkout@v4 + + - name: Fetch cached libyaml + id: cached_libyaml + uses: actions/cache/restore@v4 + with: + path: libyaml + key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} + fail-on-cache-miss: true + + - name: configure docker foreign arch support + uses: docker/setup-qemu-action@v3 + if: matrix.arch != 'x86_64' + + - name: Build/Test/Package + env: + CIBW_ARCHS: all + # HACK: ick, maybe deconstruct the matrix a bit or query cibuildwheel for its default target *linux spec first? + CIBW_BUILD: ${{matrix.spec}}-${{ contains(matrix.platform, 'musllinux') && 'musllinux' || 'manylinux' }}_${{matrix.arch}} + CIBW_BUILD_VERBOSITY: 1 + # containerized Linux builds require explicit CIBW_ENVIRONMENT + CIBW_ENVIRONMENT: > + C_INCLUDE_PATH=libyaml/include + LIBRARY_PATH=libyaml/src/.libs + LD_LIBRARY_PATH=libyaml/src/.libs + PYYAML_FORCE_CYTHON=1 + PYYAML_FORCE_LIBYAML=1 + CIBW_TEST_COMMAND: cd {project}; pytest + CIBW_TEST_REQUIRES: pytest + run: | + set -eux + python3 -V + python3 -m pip install -U --user ${{ matrix.cibw_version || 'cibuildwheel' }} + python3 -m cibuildwheel --platform auto --output-dir dist . + + echo "ARTIFACT_NAME=$(ls ./dist/)" >> "$GITHUB_OUTPUT" + # FIXME: ensure exactly one artifact + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.build.outputs.ARTIFACT_NAME }} + path: dist/*.whl + if-no-files-found: error + if: ${{ ! env.skip_artifact_upload }} + # # macos_libyaml: # name: libyaml macos ${{matrix.arch}} @@ -450,6 +476,8 @@ jobs: - name: Checkout pyyaml uses: actions/checkout@v4 + with: + python-version: 3.x - name: Get cached libyaml state id: cached_libyaml @@ -487,6 +515,8 @@ jobs: path: dist/*.whl if-no-files-found: error +# FIXME: artifact combine job? +# # check: # if: always() # needs: