Skip to content
Native

Fix rustfst-python build in various ways to support Python 3.12 among other things #1297

Sign in to view logs

Fix rustfst-python build in various ways to support Python 3.12 among other things

Fix rustfst-python build in various ways to support Python 3.12 among other things #1297

Workflow file for this run

.github/workflows/native.yml at e701bc8
name: Native
on:
pull_request:
push:
branches:
- 'main'
tags:
- 'rustfst-v**'
schedule:
- cron: '0 9 * * *'
# This codes allows to stop the CI when a new commit has been pushed on the same PR or branch.
# It doesn't apply to branches.
# Ref https://docs.github.com/en/enterprise-cloud@latest/actions/using-jobs/using-concurrency#example-using-a-fallback-value
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
env:
PYTHON_VERSION: 3.9
GIT_COMMITTER_NAME: garvys-ci-bot
GIT_COMMITTER_EMAIL: [email protected]
jobs:
python-fmt:
name: Check python formatting
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }} # Version range or exact version of a Python version to use, using SemVer's version range syntax
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- name: Check formatting
run: |
pip install black==22.3.0
python -m black --check . || fail "Format your code by running black ." 1
rust-fmt:
name: Check rust formatting
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: 1.78.0
components: rustfmt
- name: Cargo fmt check
run: cargo fmt --all -- --check
openfst:
name: Fetch or Build OpenFST
needs: [python-fmt, rust-fmt]
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-13]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- name: Cache Openfst build
id: cache
uses: actions/cache@v4
with:
path: openfst-1.7.2
key: ${{ runner.os }}-openfst-1.7.2-build
- name: Build Openfst if necessary
if: steps.cache.outputs.cache-hit != 'true'
run: ./build_openfst.sh
- uses: actions/upload-artifact@v4
with:
name: openfst-${{ runner.os }}
path: |
openfst-1.7.2/src/include/
openfst-1.7.2/lib/libfst.a
openfst-1.7.2/bin/
openfst-1.7.2/lib/
if-no-files-found: error
# generate-openfst-output:
# name: Generate OpenFST outputs on the Test Suite
# needs: openfst
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# - uses: actions/download-artifact@v3
# with:
# name: openfst-${{ runner.os }}
# path: openfst-1.7.2
# - name: Generate outputs
# run: ./run_openfst.sh
# - uses: actions/upload-artifact@v3
# with:
# name: openfst-test-data
# path: |
# rustfst-tests-data/fst_*/*
# !rustfst-tests-data/fst_*/*.h
# rustfst-tests-data/symt_*/*
# !rustfst-tests-data/symt_*/*.h
# rustfst-tests-data/weights/*.json
rust-tests:
name: rust-tests
needs: openfst
strategy:
fail-fast: false
matrix:
rust: [1.74.0, stable]
os: [ubuntu-latest, macos-13]
cargo-args:
- --all --benches --examples --bins --tests
- --manifest-path rustfst/Cargo.toml --features "state-label-u32"
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: ${{ matrix.rust }}
- uses: actions/download-artifact@v4
with:
name: openfst-${{ runner.os }}
path: openfst-1.7.2
- name: Generate outputs
run: ./run_openfst.sh
- name: Cargo test
run: cargo test ${{ matrix.cargo-args }}
rust-clippy:
name: rust-clippy
needs: openfst
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: 1.78.0
components: clippy
- name: Cargo Clippy
run: cargo clippy -- -D warnings
rust-doc:
name: rust-doc
needs: openfst
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: 1.78.0
- name: Cargo doc`
run: cargo doc --all --no-deps
rustfst-python-bench:
name: rustfst-python-bench
needs: openfst
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: stable
- uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }} # Version range or exact version of a Python version to use, using SemVer's version range syntax
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- uses: actions/download-artifact@v4
with:
name: openfst-${{ runner.os }}
path: openfst-1.7.2
- name: Generate outputs
run: ./run_openfst.sh
- name: Install Hyperfine MacOS
if: runner.os == 'macOS'
run: brew install hyperfine
- name: Install Hyperfine Linux
if: runner.os == 'Linux'
run: |
wget https://github.com/sharkdp/hyperfine/releases/download/v1.6.0/hyperfine_1.6.0_amd64.deb
sudo dpkg -i hyperfine_1.6.0_amd64.deb
- name: Build benchmark
run: ./build_bench.sh debug
- name: Test benchmark
run: |
chmod +x openfst-1.7.2/bin/fst*
python -m pip install -e rustfst-python-bench
python rustfst-python-bench/rustfst_python_bench/bench_all.py -w 1 -r 2 debug rustfst-tests-data/fst_003/raw_vector.fst report.md
python rustfst-python-bench/rustfst_python_bench/bench_all_detailed.py -w 1 -r 2 debug rustfst-tests-data/fst_003/raw_vector.fst report2.md
rustfst-python:
name: rustfst-python
needs: openfst
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-13]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: stable
- uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }} # Version range or exact version of a Python version to use, using SemVer's version range syntax
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- name: Build rustfst-python and install test tools
run: |
pip install -e ".[tests]"
- name: Test rustfst-python
run: python -m pytest -s --cache-clear --disable-warnings rustfst-python
python-doc:
name: python-doc
needs: openfst
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: stable
- uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }} # Version range or exact version of a Python version to use, using SemVer's version range syntax
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- name: Build rustfst-python and install docs tools
run: |
pip install -e ".[docs]"
- name: Test doc generation
run: |
cd rustfst-python
mkdocs build
publish-python-doc:
name: Publish rustfst-python's doc
needs: [ rust-clippy, rust-doc, rust-tests, rustfst-python-bench, rustfst-python, python-doc ]
runs-on: ubuntu-latest
if: github.event_name != 'schedule' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/rustfst-v'))
permissions:
actions: write
contents: write
deployments: write
id-token: write
packages: write
pages: write
repository-projects: write
steps:
- uses: actions/checkout@v4
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: stable
- uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- run: pip install -e ".[docs]"
- run: git config user.name ${{ env.GIT_COMMITTER_NAME }}
- run: git config user.email ${{ env.GIT_COMMITTER_EMAIL }}
- run: git fetch origin gh-pages --depth=1
- run: cd rustfst-python && mkdocs build
- name: Deploy Doc to latest
if: github.ref == 'refs/heads/main'
run: cd rustfst-python && mike deploy --push latest
- name: Retrieve tag
uses: olegtarasov/[email protected]
id: tagName
if: startsWith(github.ref, 'refs/tags/rustfst-v')
- name: Deploy Doc to tag
if: startsWith(github.ref, 'refs/tags/rustfst-v')
run: cd rustfst-python && mike deploy --push $GIT_TAG_NAME
publish-rust-crate:
name: Publish Rust Crate to Crates.io
needs: [ rust-clippy, rust-doc, rust-tests, rustfst-python-bench, rustfst-python, python-doc ]
runs-on: ubuntu-latest
if: startsWith(github.ref, 'refs/tags/rustfst-v')
steps:
- uses: actions/checkout@v4
- name: Install Rust Stable
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: stable
- name: Publish rustfst
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
run: |
cd rustfst
cargo publish
sleep 30
cd ..
- name: Publish rustfst-ffi
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
run: |
cd rustfst-ffi
cargo publish
cd ..
build-python-sdist:
name: Build source distribution
runs-on: ubuntu-latest
needs: [ rust-clippy, rust-doc, rust-tests, rustfst-python-bench, rustfst-python, python-doc ]
if: startsWith(github.ref, 'refs/tags/rustfst-v')
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build source distribution
run: pipx run build --sdist
- name: Upload source distribution as artifact
uses: actions/upload-artifact@v4
with:
name: cibw-sdist
path: dist/*.tar.gz
publish-python-wheels:
name: Publish Python Wheels to PyPI
needs: [ build-python-sdist ]
if: startsWith(github.ref, 'refs/tags/rustfst-v')
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- macos-13
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build wheels
uses: pypa/[email protected]
env:
CIBW_SKIP: cp*-musllinux_i686
CIBW_ENVIRONMENT: PATH="$HOME/.cargo/bin:$PATH"
CIBW_ENVIRONMENT_PASS_LINUX: PATH="$HOME/.cargo/bin:$PATH"
CIBW_BEFORE_BUILD: "which rustup || curl https://sh.rustup.rs -sSf | sh -s -- -y;
. $HOME/.cargo/env; rustup toolchain add stable;
rustup default stable;"
CIBW_BEFORE_BUILD_MACOS: "rustup target add aarch64-apple-darwin;"
# Build `universal2` and `arm64` wheels on an Intel runner.
# Note that the `arm64` wheel and the `arm64` part of the `universal2`
# wheel cannot be tested in this configuration.
CIBW_ARCHS_MACOS: "x86_64 arm64"
# On an Linux Intel runner with qemu installed, build Intel and ARM wheels
CIBW_ARCHS_LINUX: "auto"
- name: Upload wheels as artifacts
uses: actions/upload-artifact@v4
with:
name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
path: ./wheelhouse/*.whl
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Upload sdist and wheels to PyPI
run: |
python -m pip install twine
python -m twine upload -u "__token__" -p ${{ secrets.PYPI_PASSWORD }} -r pypi --verbose wheelhouse/* dist/*.tar.gz