revert: protoc removal (#826)

* revert: protoc removal

* ci: test run release jobs

* revert: dockerfile protoc install

* revert: deployer prepare.sh protoc install

* ci: restore ci

.circleci/config.yml

@@ -176,6 +176,7 @@ jobs:
     steps:
       - checkout
       - restore-cargo-cache
+      - install-protoc
       # Check this to make sure we do not include patched dependencies in
       # the Cargo.lock.
       - run:
@@ -206,6 +207,7 @@ jobs:
     steps:
       - checkout
       - restore-cargo-cache
+      - install-protoc
       - apply-patches
       - run: cargo fmt --all --check --manifest-path << parameters.path >>/Cargo.toml
       # TODO: this is incompatible with workspace inheritance, uncomment when
@@ -313,9 +315,14 @@ jobs:
       resource_class:
         description: "The resource type to use for the machine"
         type: string
+      protoc_arch:
+        description: "Architecture to use to install protoc"
+        type: string
     steps:
       - checkout
       - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y libssl-dev musl-tools clang
+      - install-protoc:
+          arch: << parameters.protoc_arch >>
       - run:
           name: Install Rust
           command: curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | bash -s -- -y --default-toolchain 1.68.0 --target << parameters.target >>
@@ -338,7 +345,7 @@ jobs:
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: "sparse"
     steps:
       - checkout
-      - run: choco install -y strawberryperl
+      - run: choco install -y strawberryperl protoc
       - run:
           name: Install Rust
           command: |
@@ -365,6 +372,8 @@ jobs:
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: "sparse"
     steps:
       - checkout
+      - install-protoc:
+          arch: osx-x86_64
       - run:
           name: Install Rust
           command: curl --proto '=https' https://sh.rustup.rs -sSf | bash -s -- -y --default-toolchain 1.68.0 --target x86_64-apple-darwin
@@ -433,6 +442,10 @@ jobs:
     resource_class: medium
     steps:
       - checkout
+      - run:
+          name: "Install Protobuf"
+          command: |
+            brew install protobuf
       - run:
           name: "Install Rust"
           command: |
@@ -536,6 +549,7 @@ workflows:
           name: build-binaries-x86_64-gnu
           image: ubuntu-2204:2022.04.1
           target: x86_64-unknown-linux-gnu
+          protoc_arch: linux-x86_64
           resource_class: xlarge
           requires:
             - linux-qa
@@ -546,6 +560,7 @@ workflows:
           name: build-binaries-x86_64-musl
           image: ubuntu-2204:2022.04.1
           target: x86_64-unknown-linux-musl
+          protoc_arch: linux-x86_64
           resource_class: xlarge
           requires:
             - linux-qa
@@ -557,6 +572,7 @@ workflows:
           image: ubuntu-2004:202101-01
           target: aarch64-unknown-linux-musl
           resource_class: arm.xlarge
+          protoc_arch: linux-aarch_64
           requires:
             - linux-qa
           filters:

Cargo.toml

@@ -61,8 +61,8 @@ http = "0.2.8"
 hyper = "0.14.23"
 jsonwebtoken = { version = "8.2.0" }
 once_cell = "1.16.0"
-opentelemetry = { version = "0.19.0", features = ["rt-tokio"] }
-opentelemetry-http = "0.8.0"
+opentelemetry = { version = "0.18.0", features = ["rt-tokio"] }
+opentelemetry-http = "0.7.0"
 pin-project = "1.0.12"
 pipe = "0.4.0"
 portpicker = "0.1.1"
@@ -87,8 +87,7 @@ tonic-build = "0.8.3"
 tower = "0.4.13"
 tower-http = { version = "0.4.0", features = ["trace"] }
 tracing = { version = "0.1.37", default-features = false }
-# Remove git dependency when https://github.com/tokio-rs/tracing-opentelemetry/pull/12 is merged and released
-tracing-opentelemetry = { git = "https://github.com/oddgrd/tracing-opentelemetry", branch = "update-opentelemetry-0.19" }
+tracing-opentelemetry = "0.18.0"
 tracing-subscriber = { version = "0.3.16", default-features = false, features = [
   "registry",
   "std",

Containerfile

@@ -4,6 +4,13 @@ FROM docker.io/library/rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-build
 RUN apt-get update &&\
     apt-get install -y curl
 
+# download protoc binary and unzip it in usr/bin
+ARG PROTOC_ARCH
+RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-${PROTOC_ARCH}.zip &&\
+    unzip -o protoc-21.9-linux-${PROTOC_ARCH}.zip -d /usr bin/protoc &&\
+    unzip -o protoc-21.9-linux-${PROTOC_ARCH}.zip -d /usr/ 'include/*' &&\
+    rm -f protoc-21.9-linux-${PROTOC_ARCH}.zip
+
 RUN cargo install cargo-chef
 WORKDIR /build
 
@@ -30,6 +37,14 @@ RUN cargo build --bin shuttle-${folder} $(if [ "$CARGO_PROFILE" = "release" ]; t
 
 ARG RUSTUP_TOOLCHAIN
 FROM rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-common
+RUN apt-get update &&\
+    apt-get install -y curl
+# download protoc binary and unzip it in usr/bin
+ARG PROTOC_ARCH
+RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-${PROTOC_ARCH}.zip &&\
+    unzip -o protoc-21.9-linux-${PROTOC_ARCH}.zip -d /usr/ bin/protoc &&\
+    unzip -o protoc-21.9-linux-${PROTOC_ARCH}.zip -d /usr/ 'include/*' &&\
+    rm -f protoc-21.9-linux-${PROTOC_ARCH}.zip
 RUN rustup component add rust-src
 
 COPY --from=cache /build/ /usr/src/shuttle/

Makefile

@@ -65,6 +65,12 @@ CARGO_PROFILE=debug
 RUST_LOG?=shuttle=trace,debug
 endif
 
+ARCH=$(shell uname -m)
+PROTOC_ARCH=$(ARCH)
+ifeq ($(ARCH), arm64)
+PROTOC_ARCH=aarch_64
+endif
+
 POSTGRES_EXTRA_PATH?=./extras/postgres
 POSTGRES_TAG?=14
 
@@ -154,6 +160,7 @@ down: $(DOCKER_COMPOSE_FILES)
 
 shuttle-%: ${SRC} Cargo.lock
 	$(DOCKER_BUILD) \
+		--build-arg PROTOC_ARCH=$(PROTOC_ARCH) \
 		--build-arg folder=$(*) \
 		--build-arg prepare_args=$(PREPARE_ARGS) \
 		--build-arg RUSTUP_TOOLCHAIN=$(RUSTUP_TOOLCHAIN) \

README.md

@@ -50,6 +50,8 @@ Shuttle is built for productivity, reliability and performance:
 
 ## Getting Started
 
+To run a shuttle project, you need to install protoc, check out the [protoc installation guide](https://docs.shuttle.rs/support/installing-protoc) in our docs.
+
 The `cargo-shuttle` CLI can be installed with a pre-built binary or from source with cargo.
 
 Shuttle provides pre-built binaries of the `cargo-shuttle` CLI with every release

common/Cargo.toml

@@ -23,7 +23,7 @@ jsonwebtoken = { workspace = true, optional = true }
 once_cell = { workspace = true, optional = true }
 opentelemetry = { workspace = true, optional = true }
 opentelemetry-http = { workspace = true, optional = true }
-opentelemetry-otlp = { version = "0.12.0", optional = true }
+opentelemetry-otlp = { version = "0.11.0", optional = true }
 pin-project = { workspace = true, optional = true }
 prost-types = { workspace = true, optional = true }
 reqwest = { workspace = true, optional = true }

deployer/prepare.sh

@@ -30,13 +30,6 @@ shuttle-tide = { path = "/usr/src/shuttle/services/shuttle-tide" }
 shuttle-tower = { path = "/usr/src/shuttle/services/shuttle-tower" }
 shuttle-warp = { path = "/usr/src/shuttle/services/shuttle-warp" }' > $CARGO_HOME/config.toml
 
-# Install protoc since some users may need it
-ARCH="linux-x86_64" && \
-VERSION="22.2" && \
-curl -OL "https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-$ARCH.zip" && \
-    unzip -o "protoc-$VERSION-$ARCH.zip" bin/protoc "include/*" -d /usr/local && \
-    rm -f "protoc-$VERSION-$ARCH.zip"
-
 # Add the wasm32-wasi target
 rustup target add wasm32-wasi