Add HIL testing (#1297)

* Create the `hil-test` package

* Add a simple integration test to verify basic GPIO functionality

* WIP

* feat: Update with esp-hal unification

* build: Update dependencies

* feat: Add a simple CI workflow test

* ci: Avoid using a gh-hosted-runner to build

* ci: Remove building bins in gh-hosted-runner

* ci: Remove HIL Gpio CI test

* ci: Test all the available tests

* test: Add spi_full_duplex test

* docs: Add documentation

* test: Add uart test

* style: Remove unused imports

* docs: Update wiring, document H2 VM

* ci: Enable H2 tests

* ci: Add rust-cache action

* docs: Document H2 vm

* test: Add timeout

* ci: Enable ESP32-C3 tests

* feat: Add timeouts

* feat: Add aes test

* ci: Avoid running CI workflow when we change hil-test stuff

* test: Remove warnings

* feat: Address feedback

* feat: Update features names and spi methods

* ci: Remove rust-cache action

* Update HIL to probe-rs#2292 (#1307)

* feat: Update probe-rs/embedded-test to probe-rs#2292

* feat: Remove lib

* ci: Use a matrix

* ci: Enable ESP32C3

* feat: Add a way to cfg away test for unsuported peripherals

* ci: Update trigger conditions

* feat: Update pins to make it work on s3

* feat: Changes enabling S3

* feat: Remove log feature

* feat: Adapt for rebase

* feat: Remove env

* feat: enable S3

* chore: Remove todo

* build: Pin dependencies

* feat: Add target alias

* docs: Update readme

* fix: Fix traits imports after rebase. Use debug

* build: Remove lto

* feat: Build tests on release mode

---------

Co-authored-by: Jesse Braham <[email protected]>

.github/workflows/ci.yml

@@ -18,12 +18,14 @@ on:
     paths-ignore:
       - "**/CHANGELOG.md"
       - "**/README.md"
+      - "**/hil-test/**"
   push:
     branches-ignore:
       - "gh-readonly-queue/**"
     paths-ignore:
       - "**/CHANGELOG.md"
       - "**/README.md"
+      - "**/hil-test/**"
   merge_group:
   workflow_dispatch:
 

.github/workflows/hil.yml

@@ -0,0 +1,59 @@
+name: HIL
+
+on:
+  merge_group:
+  workflow_dispatch:
+    inputs:
+      repository:
+        description: "Owner and repository to test"
+        required: true
+        default: 'esp-rs/esp-hal'
+      branch:
+        description: "Branch, tag or SHA to checkout."
+        required: true
+        default: "main"
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  # Test RISC-V targets:
+  riscv-hil:
+    name: HIL Test | ${{ matrix.target.soc }}
+    runs-on:
+      labels: [self-hosted, "${{ matrix.target.runner }}"]
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - soc: esp32c3
+            runner: rustboard
+            rust-target: riscv32imc-unknown-none-elf
+          - soc: esp32c6
+            runner: esp32c6-usb
+            rust-target: riscv32imac-unknown-none-elf
+          - soc: esp32h2
+            runner: esp32h2-usb
+            rust-target: riscv32imac-unknown-none-elf
+    steps:
+      - uses: actions/checkout@v4
+        if: github.event_name != 'workflow_dispatch'
+
+      - uses: actions/checkout@v4
+        if: github.event_name == 'workflow_dispatch'
+        with:
+          repository: ${{ github.event.inputs.repository }}
+          ref: ${{ github.event.inputs.branch }}
+
+      - uses: dtolnay/rust-toolchain@v1
+        with:
+          target: ${{ matrix.target.rust-target }}
+          toolchain: nightly
+          components: rust-src
+
+      - name: Run tests
+        working-directory: hil-test
+        run: cargo ${{ matrix.target.soc }}
+
+  # Test Xtensa targets:
+  # TODO: Add jobs for Xtensa once supported by `probe-rs`

Cargo.toml

@@ -9,4 +9,5 @@ exclude  = [
     "esp-metadata",
     "esp-riscv-rt",
     "examples",
+    "hil-test",
 ]

hil-test/.cargo/config.toml

@@ -0,0 +1,29 @@
+[alias]
+# esp32   = "test --release --features=esp32   --target=xtensa-esp32-none-elf -- --chip esp32-3.3v"
+# esp32c2 = "test --release --features=esp32c2 --target=riscv32imc-unknown-none-elf -- --chip esp32c2"
+esp32c3 = "test --release --features=esp32c3 --target=riscv32imc-unknown-none-elf -- --chip esp32c3"
+esp32c6 = "test --release --features=esp32c6 --target=riscv32imac-unknown-none-elf -- --chip esp32c6"
+esp32h2 = "test --release --features=esp32h2 --target=riscv32imac-unknown-none-elf -- --chip esp32h2"
+# esp32p4 = "test --release --features=esp32p4 --target=riscv32imafc-unknown-none-elf -- --chip esp32p4"
+# esp32s2 = "test --release --features=esp32s2 --target=xtensa-esp32s2-none-elf -- --chip esp32s2"
+esp32s3 = "test --release --features=esp32s3 --target=xtensa-esp32s3-none-elf -- --chip esp32s3"
+
+[target.'cfg(target_arch = "riscv32")']
+runner    = "probe-rs run"
+rustflags = [
+    "-C", "link-arg=-Tlinkall.x",
+    "-C", "link-arg=-Tembedded-test.x",
+    "-C", "link-arg=-Tdefmt.x",
+]
+
+[target.'cfg(target_arch = "xtensa")']
+runner    = "probe-rs run"
+rustflags = [
+    "-C", "link-arg=-nostartfiles",
+    "-C", "link-arg=-Wl,-Tlinkall.x",
+    "-C", "link-arg=-Tdefmt.x",
+    "-C", "link-arg=-Tembedded-test.x",
+]
+
+[unstable]
+build-std = ["core"]

hil-test/Cargo.toml

@@ -0,0 +1,93 @@
+[package]
+name    = "hil-test"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+[[test]]
+name    = "aes"
+harness = false
+
+[[test]]
+name    = "gpio"
+harness = false
+
+[[test]]
+name    = "spi_full_duplex"
+harness = false
+
+[[test]]
+name    = "uart"
+harness = false
+
+[dependencies]
+defmt = { version = "0.3.5" }
+defmt-rtt = { version = "0.4.0" }
+esp-hal = { path = "../esp-hal", features = ["embedded-hal", "embedded-hal-02",  "defmt"], optional = true }
+embedded-hal-02       = { version = "0.2.7", package = "embedded-hal", features = ["unproven"] }
+embedded-hal-async = { version = "1.0.0", optional = true }
+embedded-hal     = { version = "1.0.0" }
+embedded-hal-nb      = { version = "1.0.0", optional = true }
+embassy-executor = { default-features = false, version = "0.5.0", features = ["executor-thread", "arch-riscv32"], optional = true }
+semihosting = "0.1.6"
+
+[dev-dependencies]
+# Add the `embedded-test/defmt` feature for more verbose testing
+embedded-test = {git = "https://github.com/probe-rs/embedded-test", rev = "8e3f925"}
+
+[features]
+# Device support (required!):
+esp32   = ["esp-hal/esp32"]
+esp32c2 = ["esp-hal/esp32c2"]
+esp32c3 = ["esp-hal/esp32c3"]
+esp32c6 = ["esp-hal/esp32c6"]
+esp32h2 = ["esp-hal/esp32h2"]
+esp32s2 = ["esp-hal/esp32s2"]
+esp32s3 = ["esp-hal/esp32s3"]
+# Async & Embassy:
+async = ["dep:embedded-hal-async", "esp-hal?/async"]
+embassy = ["esp-hal?/embassy", "embedded-test/embassy", "dep:embassy-executor"]
+embassy-executor-interrupt = ["esp-hal?/embassy-executor-interrupt"]
+embassy-executor-thread    = ["esp-hal?/embassy-executor-thread"]
+embassy-time-systick-16mhz = ["esp-hal?/embassy-time-systick-16mhz"]
+embassy-time-systick-80mhz = ["esp-hal?/embassy-time-systick-80mhz"]
+embassy-time-timg0   = ["esp-hal?/embassy-time-timg0"]
+
+# cargo build/run
+[profile.dev]
+codegen-units    = 1
+debug            = 2
+debug-assertions = true  # <-
+incremental      = false
+opt-level        = 'z'   # <-
+overflow-checks  = true  # <-
+
+# cargo test
+[profile.test]
+codegen-units    = 1
+debug            = 2
+debug-assertions = true  # <-
+incremental      = false
+opt-level        = 3     # <-
+overflow-checks  = true  # <-
+
+# cargo build/run --release
+[profile.release]
+codegen-units    = 1
+debug            = 2
+debug-assertions = false # <-
+incremental      = false
+opt-level        = 3     # <-
+overflow-checks  = false # <-
+
+# cargo test --release
+[profile.bench]
+codegen-units    = 1
+debug            = 2
+debug-assertions = false # <-
+incremental      = false
+opt-level        = 3     # <-
+overflow-checks  = false # <-
+
+[patch.crates-io]
+semihosting = { git = "https://github.com/taiki-e/semihosting", rev = "c829c19" }

hil-test/README.md

@@ -0,0 +1,95 @@
+# hil-test
+
+Hardware-in-loop testing for `esp-hal`.
+
+For assistance with this package please [open an issue] or [start a discussion].
+
+[open an issue]: https://github.com/esp-rs/esp-hal/issues/new
+[start a discussion]: https://github.com/esp-rs/esp-hal/discussions/new/choose
+
+## Quickstart
+
+We use [embedded-test] as our testing framework, which relies on [defmt] internally. This allows us to write unit and integration tests much in the same way you would for a normal Rust project, when the standard library is available, and to execute them using Cargo's built-in test runner.
+
+[embedded-test]: https://github.com/probe-rs/embedded-test
+[defmt]: https://github.com/knurling-rs/defmt
+
+### Running Tests Locally
+
+We use [probe-rs] for flashing and running the tests on a target device, however, this **MUST** be installed from the correct revision, and with the correct features enabled:
+
+```text
+cargo install probe-rs \
+  --git=https://github.com/probe-rs/probe-rs \
+  --rev=b431b24 \
+  --features=cli \
+  --bin=probe-rs
+```
+
+Target device **MUST** connected via its USB-Serial-JTAG port, or if unavailable (eg. ESP32, ESP32-C2, ESP32-S2) then you must connect a compatible debug probe such as an [ESP-Prog].
+
+You can run all test for a given device using:
+
+```shell
+cargo +nightly esp32c6
+# or
+cargo +esp esp32s3
+```
+
+For running a single test on a target:
+
+```shell
+# Run GPIO tests for ESP32-C6
+CARGO_BUILD_TARGET=riscv32imac-unknown-none-elf \
+PROBE_RS_CHIP=esp32c6 \
+  cargo +nightly test --features=esp32c6 --test=gpio
+```
+- If the `--test` argument is omitted, then all tests will be run.
+- The build target **MUST** be specified via the `CARGO_BUILD_TARGET` environment variable or as an argument (`--target`).
+- The chip **MUST** be specified via the `PROBE_RS_CHIP` environment variable or as an argument of `probe-rs` (`--chip`).
+
+Some tests will require physical connections, please see the current [configuration in our runners](#running-tests-remotes-ie-on-self-hosted-runners).
+
+### Running Tests Remotes (ie. On Self-Hosted Runners)
+The [`hil.yml`] workflow builds the test suite for all our available targets and executes them.
+
+Our Virtual Machines have the following setup:
+- ESP32-C3 (`rustboard`):
+  - Devkit: `ESP32-C3-DevKit-RUST-1` connected via USB-Serial-JTAG.
+    - `GPIO2` and `GPIO4` are connected.
+  - VM: Configured with the following [setup](#vm-setup)
+- ESP32-C6 (`esp32c6-usb`):
+  - Devkit: `ESP32-C6-DevKitC-1 V1.2` connected via USB-Serial-JTAG (`USB` port).
+    - `GPIO2` and `GPIO4` are connected.
+  - VM: Configured with the following [setup](#vm-setup)
+- ESP32-H2 (`esp32h2-usb`):
+  - Devkit: `ESP32-H2-DevKitM-1` connected via USB-Serial-JTAG (`USB` port).
+    - `GPIO2` and `GPIO4` are connected.
+  - VM: Configured with the following [setup](#vm-setup)
+
+[`hil.yml`]: https://github.com/esp-rs/esp-hal/blob/main/.github/workflows/hil.yml
+
+#### VM Setup
+```bash
+# Install Rust:
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain stable -y --profile minimal
+# Source the current shell:
+source "$HOME/.cargo/env"
+# Install dependencies
+sudo apt install -y pkg-config libudev-dev
+# Install probe-rs
+cargo install probe-rs --git=https://github.com/probe-rs/probe-rs --rev=b431b24 --features=cli --bin=probe-rs --locked --force
+# Add the udev rules
+wget -O - https://probe.rs/files/69-probe-rs.rules | sudo tee /etc/udev/rules.d/69-probe-rs.rules > /dev/null
+# Add the user to plugdev group
+sudo usermod -a -G plugdev $USER
+# Reboot the VM
+```
+
+## Adding New Tests
+
+1. Create a new integration test file (`tests/$PERIPHERAL.rs`)
+2. Add a corresponding `[[test]]` entry to `Cargol.toml` (**MUST** set `harness = false`)
+3. Write the tests
+4. Document any necessary physical connections on boards connected to self-hosted runners
+5. Write some documentation at the top of the `tests/$PERIPHERAL.rs` file with the pins being used and the required connections, if applicable.