build(deps,ci): add protoc feature to build protoc from source (#356

)

* build(deps): add protoc feature that depends on protobuf-src, and use it in docs.rs

* build(deps): also add protoc feature to py, c, and tests crates

* build(ci): update Rust workflow to use protoc feature, using vendored protoc

Also test with and without vendored protoc

* build: update version diff template

.github/workflows/rust.yml

@@ -14,11 +14,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - uses: arduino/setup-protoc@v3
-        with:
-          version: "28.3"
-          # avoid rate-limiting
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
       - name: Check
@@ -35,19 +30,23 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - uses: arduino/setup-protoc@v3
-        with:
-          version: "28.3"
-          # avoid rate-limiting
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: dtolnay/rust-toolchain@stable
       - uses: actions/setup-python@v5
         with:
           python-version: "3.x"
         if: ${{ matrix.os == 'ubuntu-latest' }}
       - uses: Swatinem/rust-cache@v2
-      - name: Run unit tests
-        run: cargo test --all-features
+      - name: Run unit tests with vendored protoc
+        run: cargo test --all-features  # Uses vendored protoc; not compatible with windows
+        if: ${{ matrix.os != 'windows-latest' }}
+      - uses: arduino/setup-protoc@v3 # Download and 'install' protoc
+        with:
+          version: "28.3"
+          # avoid rate-limiting
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run unit tests with system protoc
+        run: cargo test
+        if: ${{ matrix.os != 'windows-latest' }}
       - name: Install test runner dependencies
         run: python3 -m pip install --upgrade pip protobuf pyyaml click
         if: ${{ matrix.os == 'ubuntu-latest' }}
@@ -77,11 +76,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - uses: arduino/setup-protoc@v3
-        with:
-          version: "28.3"
-          # avoid rate-limiting
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy
@@ -96,11 +90,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - uses: arduino/setup-protoc@v3
-        with:
-          version: "28.3"
-          # avoid rate-limiting
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
       - name: Doc
@@ -116,11 +105,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - uses: arduino/setup-protoc@v3
-        with:
-          version: "28.3"
-          # avoid rate-limiting
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: dtolnay/rust-toolchain@stable
       - name: Fetch Substrait submodule tags
         working-directory: substrait
@@ -139,4 +123,4 @@ jobs:
         working-directory: rs
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-        run: cargo publish --allow-dirty
+        run: cargo publish --allow-dirty --features protoc

c/Cargo.toml

@@ -8,6 +8,10 @@ license = "Apache-2.0"
 crate-type = ["cdylib", "staticlib"]
 doc = false
 
+[features]
+# Build and vendor protoc from source. See also rs/Cargo.toml.
+protoc = ["substrait-validator/protoc"]
+
 [build-dependencies]
 cbindgen = "0.27.0"
 

ci/version-diff-template

@@ -1,5 +1,5 @@
 diff --git a/c/Cargo.toml b/c/Cargo.toml
-index d8ad49b..4bd02ef 100644
+index 114b0cc..fc9ce3e 100644
 --- a/c/Cargo.toml
 +++ b/c/Cargo.toml
 @@ -1,6 +1,6 @@
@@ -10,7 +10,7 @@ index d8ad49b..4bd02ef 100644
  edition = "2021"
  license = "Apache-2.0"
 
-@@ -12,7 +12,7 @@ doc = false
+@@ -16,7 +16,7 @@ protoc = ["substrait-validator/protoc"]
  cbindgen = "0.27.0"
 
  [dependencies]
@@ -20,7 +20,7 @@ index d8ad49b..4bd02ef 100644
  thiserror = "2.0"
  once_cell = "1.19"
 diff --git a/derive/Cargo.toml b/derive/Cargo.toml
-index 1cf1f7e..482c127 100644
+index b4ba0fc..7c24520 100644
 --- a/derive/Cargo.toml
 +++ b/derive/Cargo.toml
 @@ -4,7 +4,7 @@ description = "Procedural macros for substrait-validator"
@@ -33,7 +33,7 @@ index 1cf1f7e..482c127 100644
  license = "Apache-2.0"
 
 diff --git a/py/Cargo.toml b/py/Cargo.toml
-index d28af20..28abfb1 100644
+index 1ea9db1..027b71d 100644
 --- a/py/Cargo.toml
 +++ b/py/Cargo.toml
 @@ -1,6 +1,6 @@
@@ -44,7 +44,7 @@ index d28af20..28abfb1 100644
  edition = "2018"
  license = "Apache-2.0"
  include = [
-@@ -29,7 +29,7 @@ name = "substrait_validator"
+@@ -33,7 +33,7 @@ name = "substrait_validator"
  doc = false
 
  [dependencies]
@@ -54,7 +54,7 @@ index d28af20..28abfb1 100644
 
  [build-dependencies]
 diff --git a/py/pyproject.toml b/py/pyproject.toml
-index 6e602e6..dd144de 100644
+index b3b3464..dd144de 100644
 --- a/py/pyproject.toml
 +++ b/py/pyproject.toml
 @@ -5,7 +5,7 @@ backend-path = ["."]
@@ -67,7 +67,7 @@ index 6e602e6..dd144de 100644
  readme = "README.md"
  license = {{ file = "LICENSE" }}
 diff --git a/rs/Cargo.toml b/rs/Cargo.toml
-index 4144f94..791db63 100644
+index 1f9f762..84f67fa 100644
 --- a/rs/Cargo.toml
 +++ b/rs/Cargo.toml
 @@ -4,7 +4,7 @@ description = "Substrait validator"
@@ -79,7 +79,7 @@ index 4144f94..791db63 100644
  edition = "2021"
  license = "Apache-2.0"
  include = ["src", "build.rs", "README.md"]
-@@ -24,7 +24,7 @@ prost-types = "0.13.3"
+@@ -29,7 +29,7 @@ prost-types = "0.13.4"
 
  # Prost doesn't generate any introspection stuff, so we hack that stuff in with
  # our own procedural macros.
@@ -89,7 +89,7 @@ index 4144f94..791db63 100644
  # Google/protobuf has a funny idea about case conventions (it converts them all
  # over the place) and prost remaps to Rust's conventions to boot. So, to
 diff --git a/rs/README.md b/rs/README.md
-index 14f8216..5d908fe 100644
+index afead27..5d908fe 100644
 --- a/rs/README.md
 +++ b/rs/README.md
 @@ -6,7 +6,7 @@ plans.
@@ -111,7 +111,7 @@ index 14f8216..5d908fe 100644
 
  This adds the `substrait_validator::Config::add_curl_yaml_uri_resolver()`
 diff --git a/tests/Cargo.toml b/tests/Cargo.toml
-index cf1ba03..4f6c8d6 100644
+index 62d2f87..79cb4d5 100644
 --- a/tests/Cargo.toml
 +++ b/tests/Cargo.toml
 @@ -1,6 +1,6 @@
@@ -122,8 +122,8 @@ index cf1ba03..4f6c8d6 100644
  edition = "2018"
  license = "Apache-2.0"
  default-run = "runner"
-@@ -14,7 +14,7 @@ name = "find_protoc"
- path = "src/find_protoc.rs"
+@@ -18,7 +18,7 @@ path = "src/find_protoc.rs"
+ protoc = ["dep:protobuf-src", "substrait-validator/protoc"]
 
  [dependencies]
 -substrait-validator = {{ path = "../rs", version = "{frm}" }}

py/Cargo.toml

@@ -16,6 +16,10 @@ include = [
     "/tests",
 ]
 
+[features]
+# Build and vendor protoc from source. See also rs/Cargo.toml.
+protoc = ["dep:protobuf-src", "substrait-validator/protoc"]
+
 [lib]
 crate-type = ["cdylib"]
 
@@ -37,3 +41,6 @@ prost-build = "0.13.4"
 pyo3-build-config = "0.23.3"
 walkdir = "2"
 dunce = "1"
+
+# Used in the 'protoc' feature.
+protobuf-src = { version = "2.1", optional = true }

py/build.rs

@@ -78,7 +78,19 @@ fn main() {
     fs::create_dir_all(&intermediate_path).expect("failed to create protoc output directory");
 
     // Run protoc.
-    let mut cmd = Command::new(prost_build::protoc_from_env());
+    let protoc_path: PathBuf;
+    #[cfg(feature = "protoc")]
+    {
+        // Use vendored protobuf compiler if requested.
+        protoc_path = protobuf_src::protoc();
+        println!("cargo:warning=Using vendored protoc: {:?}", protoc_path);
+    }
+    #[cfg(not(feature = "protoc"))]
+    {
+        protoc_path = prost_build::protoc_from_env();
+    }
+
+    let mut cmd = Command::new(protoc_path);
     for input_path in input_paths.iter() {
         let mut proto_path_arg = OsString::new();
         proto_path_arg.push("--proto_path=");

rs/Cargo.toml

@@ -16,6 +16,11 @@ include = ["src", "build.rs", "README.md"]
 # functionally change anything.
 private_docs = []
 
+# Build and vendor protoc from source. This is useful to have a consistent
+# version of protoc across platforms/build systems. In particular, docs.rs uses
+# it.
+protoc = ["dep:protobuf-src"]
+
 [dependencies]
 
 # Prost is used to deal with protobuf serialization and deserialization.
@@ -93,5 +98,11 @@ semver = "1.0"
 # Used for generating Rust structs from the protobuf definitions.
 prost-build = "0.13.4"
 
+# Used in the 'protoc' feature.
+protobuf-src = { version = "2.1", optional = true }
+
 # Used to automatically find all protobuf files.
 walkdir = "2"
+
+[package.metadata.docs.rs]
+features = ["protoc"]

rs/build.rs

@@ -129,6 +129,10 @@ fn main() -> Result<()> {
         }
     }
 
+    #[cfg(feature = "protoc")]
+    // Use vendored protobuf compiler if requested.
+    std::env::set_var("PROTOC", protobuf_src::protoc());
+
     // Find all protobuf files in our resource directory. We just synchronized
     // these files if we're building from git.
     let proto_path = PathBuf::from(&resource_dir).join("proto");

tests/Cargo.toml

@@ -13,6 +13,10 @@ path = "src/runner.rs"
 name = "find_protoc"
 path = "src/find_protoc.rs"
 
+[features]
+# Build and vendor protoc from source. See also rs/Cargo.toml.
+protoc = ["dep:protobuf-src", "substrait-validator/protoc"]
+
 [dependencies]
 substrait-validator = { path = "../rs", version = "0.1.1" }
 serde = { version = "1.0", features = ["derive"] }
@@ -21,3 +25,5 @@ walkdir = "2"
 glob = "0.3"
 prost-build = "0.13.4"
 rayon = "1.10"
+# Used in the 'protoc' feature.
+protobuf-src = { version = "2.1", optional = true }

tests/src/find_protoc.rs

@@ -3,6 +3,20 @@
 //! Prost has some magic for finding the path to protoc, so let's use that in
 //! the Python code as well...
 
+use std::path::PathBuf;
+
 fn main() {
-    println!("{}", prost_build::protoc_from_env().display());
+    let protoc_path: PathBuf;
+    #[cfg(feature = "protoc")]
+    {
+        // Use vendored protobuf compiler if requested.
+        protoc_path = protobuf_src::protoc();
+        println!("cargo:warning=Using vendored protoc: {:?}", protoc_path);
+    }
+    #[cfg(not(feature = "protoc"))]
+    {
+        protoc_path = prost_build::protoc_from_env();
+    }
+
+    println!("{}", protoc_path.display());
 }