diff --git a/.circleci/config.yml b/.circleci/config.yml index ddb2bf146..6f894fa3d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -176,6 +176,7 @@ jobs: steps: - checkout - restore-cargo-cache + - install-protoc # Check this to make sure we do not include patched dependencies in # the Cargo.lock. - run: @@ -206,6 +207,7 @@ jobs: steps: - checkout - restore-cargo-cache + - install-protoc - apply-patches - run: cargo fmt --all --check --manifest-path << parameters.path >>/Cargo.toml # TODO: this is incompatible with workspace inheritance, uncomment when @@ -313,9 +315,14 @@ jobs: resource_class: description: "The resource type to use for the machine" type: string + protoc_arch: + description: "Architecture to use to install protoc" + type: string steps: - checkout - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y libssl-dev musl-tools clang + - install-protoc: + arch: << parameters.protoc_arch >> - run: name: Install Rust command: curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | bash -s -- -y --default-toolchain 1.68.0 --target << parameters.target >> @@ -338,7 +345,7 @@ jobs: CARGO_REGISTRIES_CRATES_IO_PROTOCOL: "sparse" steps: - checkout - - run: choco install -y strawberryperl + - run: choco install -y strawberryperl protoc - run: name: Install Rust command: | @@ -365,6 +372,8 @@ jobs: CARGO_REGISTRIES_CRATES_IO_PROTOCOL: "sparse" steps: - checkout + - install-protoc: + arch: osx-x86_64 - run: name: Install Rust command: curl --proto '=https' https://sh.rustup.rs -sSf | bash -s -- -y --default-toolchain 1.68.0 --target x86_64-apple-darwin @@ -433,6 +442,10 @@ jobs: resource_class: medium steps: - checkout + - run: + name: "Install Protobuf" + command: | + brew install protobuf - run: name: "Install Rust" command: | @@ -536,6 +549,7 @@ workflows: name: build-binaries-x86_64-gnu image: ubuntu-2204:2022.04.1 target: x86_64-unknown-linux-gnu + protoc_arch: linux-x86_64 resource_class: xlarge requires: - linux-qa @@ -546,6 +560,7 @@ workflows: name: build-binaries-x86_64-musl image: ubuntu-2204:2022.04.1 target: x86_64-unknown-linux-musl + protoc_arch: linux-x86_64 resource_class: xlarge requires: - linux-qa @@ -557,6 +572,7 @@ workflows: image: ubuntu-2004:202101-01 target: aarch64-unknown-linux-musl resource_class: arm.xlarge + protoc_arch: linux-aarch_64 requires: - linux-qa filters: diff --git a/Cargo.lock b/Cargo.lock index db17858cf..a93ca732c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3706,9 +3706,9 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.19.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4b8347cc26099d3aeee044065ecc3ae11469796b4d65d065a23a584ed92a6f" +checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" dependencies = [ "opentelemetry_api", "opentelemetry_sdk", @@ -3716,9 +3716,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.8.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a819b71d6530c4297b49b3cae2939ab3a8cc1b9f382826a1bc29dd0ca3864906" +checksum = "1edc79add46364183ece1a4542592ca593e6421c60807232f5b8f7a31703825d" dependencies = [ "async-trait", "bytes", @@ -3728,9 +3728,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.12.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8af72d59a4484654ea8eb183fea5ae4eb6a41d7ac3e3bae5f4d2a282a3a7d3ca" +checksum = "d1c928609d087790fc936a1067bdc310ae702bdf3b090c3f281b713622c8bbde" dependencies = [ "async-trait", "futures", @@ -3746,38 +3746,39 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.2.0" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "045f8eea8c0fa19f7d48e7bc3128a39c2e5c533d5c61298c548dfefc1064474c" +checksum = "d61a2f56df5574508dd86aaca016c917489e589ece4141df1b5e349af8d66c28" dependencies = [ "futures", "futures-util", "opentelemetry", "prost", "tonic", + "tonic-build", ] [[package]] name = "opentelemetry_api" -version = "0.19.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed41783a5bf567688eb38372f2b7a8530f5a607a4b49d38dd7573236c23ca7e2" +checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" dependencies = [ "fnv", "futures-channel", "futures-util", "indexmap", + "js-sys", "once_cell", "pin-project-lite", "thiserror", - "urlencoding", ] [[package]] name = "opentelemetry_sdk" -version = "0.19.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b3a2a91fdbfdd4d212c0dcc2ab540de2c2bcbbd90be17de7a7daf8822d010c1" +checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" dependencies = [ "async-trait", "crossbeam-channel", @@ -6308,8 +6309,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.19.0" -source = "git+https://github.com/oddgrd/tracing-opentelemetry?branch=update-opentelemetry-0.19#76b0a0fe45fd53c352e8bfdcdfad775aed727e4c" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" dependencies = [ "once_cell", "opentelemetry", diff --git a/Cargo.toml b/Cargo.toml index deeae7fc3..9b4dfc0e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,8 +61,8 @@ http = "0.2.8" hyper = "0.14.23" jsonwebtoken = { version = "8.2.0" } once_cell = "1.16.0" -opentelemetry = { version = "0.19.0", features = ["rt-tokio"] } -opentelemetry-http = "0.8.0" +opentelemetry = { version = "0.18.0", features = ["rt-tokio"] } +opentelemetry-http = "0.7.0" pin-project = "1.0.12" pipe = "0.4.0" portpicker = "0.1.1" @@ -87,8 +87,7 @@ tonic-build = "0.8.3" tower = "0.4.13" tower-http = { version = "0.4.0", features = ["trace"] } tracing = { version = "0.1.37", default-features = false } -# Remove git dependency when https://github.com/tokio-rs/tracing-opentelemetry/pull/12 is merged and released -tracing-opentelemetry = { git = "https://github.com/oddgrd/tracing-opentelemetry", branch = "update-opentelemetry-0.19" } +tracing-opentelemetry = "0.18.0" tracing-subscriber = { version = "0.3.16", default-features = false, features = [ "registry", "std", diff --git a/Containerfile b/Containerfile index 587c48575..2ed48b613 100644 --- a/Containerfile +++ b/Containerfile @@ -4,6 +4,13 @@ FROM docker.io/library/rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-build RUN apt-get update &&\ apt-get install -y curl +# download protoc binary and unzip it in usr/bin +ARG PROTOC_ARCH +RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-${PROTOC_ARCH}.zip &&\ + unzip -o protoc-21.9-linux-${PROTOC_ARCH}.zip -d /usr bin/protoc &&\ + unzip -o protoc-21.9-linux-${PROTOC_ARCH}.zip -d /usr/ 'include/*' &&\ + rm -f protoc-21.9-linux-${PROTOC_ARCH}.zip + RUN cargo install cargo-chef WORKDIR /build @@ -30,6 +37,14 @@ RUN cargo build --bin shuttle-${folder} $(if [ "$CARGO_PROFILE" = "release" ]; t ARG RUSTUP_TOOLCHAIN FROM rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-common +RUN apt-get update &&\ + apt-get install -y curl +# download protoc binary and unzip it in usr/bin +ARG PROTOC_ARCH +RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-${PROTOC_ARCH}.zip &&\ + unzip -o protoc-21.9-linux-${PROTOC_ARCH}.zip -d /usr/ bin/protoc &&\ + unzip -o protoc-21.9-linux-${PROTOC_ARCH}.zip -d /usr/ 'include/*' &&\ + rm -f protoc-21.9-linux-${PROTOC_ARCH}.zip RUN rustup component add rust-src COPY --from=cache /build/ /usr/src/shuttle/ diff --git a/Makefile b/Makefile index 3b86aa88d..193c96537 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,12 @@ CARGO_PROFILE=debug RUST_LOG?=shuttle=trace,debug endif +ARCH=$(shell uname -m) +PROTOC_ARCH=$(ARCH) +ifeq ($(ARCH), arm64) +PROTOC_ARCH=aarch_64 +endif + POSTGRES_EXTRA_PATH?=./extras/postgres POSTGRES_TAG?=14 @@ -154,6 +160,7 @@ down: $(DOCKER_COMPOSE_FILES) shuttle-%: ${SRC} Cargo.lock $(DOCKER_BUILD) \ + --build-arg PROTOC_ARCH=$(PROTOC_ARCH) \ --build-arg folder=$(*) \ --build-arg prepare_args=$(PREPARE_ARGS) \ --build-arg RUSTUP_TOOLCHAIN=$(RUSTUP_TOOLCHAIN) \ diff --git a/README.md b/README.md index 72166087a..b78686757 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,8 @@ Shuttle is built for productivity, reliability and performance: ## Getting Started +To run a shuttle project, you need to install protoc, check out the [protoc installation guide](https://docs.shuttle.rs/support/installing-protoc) in our docs. + The `cargo-shuttle` CLI can be installed with a pre-built binary or from source with cargo. Shuttle provides pre-built binaries of the `cargo-shuttle` CLI with every release diff --git a/common/Cargo.toml b/common/Cargo.toml index 90d9fbe8d..17d73e2f2 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -23,7 +23,7 @@ jsonwebtoken = { workspace = true, optional = true } once_cell = { workspace = true, optional = true } opentelemetry = { workspace = true, optional = true } opentelemetry-http = { workspace = true, optional = true } -opentelemetry-otlp = { version = "0.12.0", optional = true } +opentelemetry-otlp = { version = "0.11.0", optional = true } pin-project = { workspace = true, optional = true } prost-types = { workspace = true, optional = true } reqwest = { workspace = true, optional = true } diff --git a/deployer/prepare.sh b/deployer/prepare.sh index 2ca807cc8..09b6a2b09 100755 --- a/deployer/prepare.sh +++ b/deployer/prepare.sh @@ -30,13 +30,6 @@ shuttle-tide = { path = "/usr/src/shuttle/services/shuttle-tide" } shuttle-tower = { path = "/usr/src/shuttle/services/shuttle-tower" } shuttle-warp = { path = "/usr/src/shuttle/services/shuttle-warp" }' > $CARGO_HOME/config.toml -# Install protoc since some users may need it -ARCH="linux-x86_64" && \ -VERSION="22.2" && \ -curl -OL "https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-$ARCH.zip" && \ - unzip -o "protoc-$VERSION-$ARCH.zip" bin/protoc "include/*" -d /usr/local && \ - rm -f "protoc-$VERSION-$ARCH.zip" - # Add the wasm32-wasi target rustup target add wasm32-wasi