Merge pull request #4 from 0xPolygonZero/develop

fix: docker build for worker and leader (0xPolygonZero#329)

.github/workflows/docker_build.yml

@@ -0,0 +1,35 @@
+name: Docker Build & Run
+
+on:
+  push:
+    branches: [develop, main]
+  pull_request:
+    branches:
+      - "**"
+  workflow_dispatch:
+    branches:
+      - "**"
+
+jobs:
+  docker:
+    name: Build and run leader and worker docker images for regression check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build leader docker container
+        run: |
+          docker build --progress plain -t leader:${{ github.ref_name }} -f leader.Dockerfile .
+
+      - name: Run leader docker container
+        run: |
+          docker run --rm leader:${{ github.ref_name }} --help
+
+      - name: Build worker docker container
+        run: |
+          docker build --progress plain -t worker:${{ github.ref_name }} -f worker.Dockerfile .
+
+      - name: Run worker docker container
+        run: |
+          docker run --rm worker:${{ github.ref_name }} --help

.github/workflows/docker_build_push.yml

@@ -0,0 +1,83 @@
+name: Docker Build & Push
+
+on:
+  push:
+    branches: [develop, main]
+  release:
+    types: [created]
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME_LEADER: ${{ github.repository }}-leader
+  IMAGE_NAME_WORKER: ${{ github.repository }}-worker
+
+jobs:
+  docker:
+    name: Build and push leader and worker docker images to GitHub Container Registry
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Leader Docker
+        id: meta_leader
+        uses: docker/metadata-action@v5
+        with:
+          images: | 
+            name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LEADER }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Push to GitHub Container Registry - Leader
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./leader.Dockerfile
+          push: true
+          # platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta_leader.outputs.tags }}
+          labels: ${{ steps.meta_leader.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Extract metadata (tags, labels) for Worker Docker
+        id: meta_worker
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_WORKER }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Push to GitHub Container Registry - Worker
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./worker.Dockerfile
+          push: true
+          # platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta_worker.outputs.tags }}
+          labels: ${{ steps.meta_worker.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

leader.Dockerfile

@@ -0,0 +1,60 @@
+FROM rustlang/rust:nightly-bullseye-slim as builder
+
+RUN apt-get update && apt-get install -y libjemalloc2 libjemalloc-dev make libssl-dev pkg-config
+
+RUN mkdir -p zero_bin
+COPY Cargo.toml .
+# Cleanup all workspace members and add selected crates again
+RUN sed -i '/members =/{:a;N;/]/!ba};//d' Cargo.toml
+RUN sed -i 's#\[workspace\]#\[workspace\]\nmembers = \["zero_bin\/leader", "zero_bin\/prover", "zero_bin\/rpc", "zero_bin\/common", \
+ "zero_bin\/ops"\, "evm_arithmetization", "trace_decoder", "mpt_trie", "proc_macro", "compat"\]#' Cargo.toml
+COPY Cargo.lock .
+COPY ./rust-toolchain.toml ./
+RUN cat ./Cargo.toml
+COPY ./.env ./.env
+
+COPY proof_gen proof_gen
+COPY mpt_trie mpt_trie
+COPY proc_macro proc_macro
+COPY compat compat
+COPY trace_decoder trace_decoder
+COPY evm_arithmetization evm_arithmetization
+COPY zero_bin/common zero_bin/common
+COPY zero_bin/ops zero_bin/ops
+COPY zero_bin/rpc zero_bin/rpc
+COPY zero_bin/prover zero_bin/prover
+COPY zero_bin/leader zero_bin/leader
+
+
+RUN \
+  touch zero_bin/common/src/lib.rs && \
+  touch zero_bin/ops/src/lib.rs && \
+  touch zero_bin/leader/src/main.rs && \
+  touch zero_bin/rpc/src/lib.rs && \
+  touch zero_bin/prover/src/lib.rs && \
+  touch evm_arithmetization/src/lib.rs && \
+  touch trace_decoder/src/lib.rs && \
+  touch mpt_trie/src/lib.rs && \
+  touch proc_macro/src/lib.rs && \
+  touch compat/src/lib.rs
+
+# Disable the lld linker for now, as it's causing issues with the linkme package.
+# https://github.com/rust-lang/rust/pull/124129
+# https://github.com/dtolnay/linkme/pull/88
+ENV RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld'
+
+RUN cargo build --release --bin leader
+RUN cargo build --release --bin rpc
+
+
+FROM debian:bullseye-slim
+RUN apt-get update && apt-get install -y ca-certificates libjemalloc2
+COPY --from=builder ./target/release/leader /usr/local/bin/leader
+COPY --from=builder ./target/release/rpc /usr/local/bin/rpc
+COPY --from=builder ./.env /.env
+
+# Workaround for the issue with the Cargo.lock search path
+# Related to issue https://github.com/0xPolygonZero/zk_evm/issues/311
+RUN mkdir -p zero_bin/leader
+
+ENTRYPOINT ["/usr/local/bin/leader"]

worker.Dockerfile

@@ -0,0 +1,40 @@
+FROM rustlang/rust:nightly-bullseye-slim as builder
+
+RUN apt-get update && apt-get install -y libjemalloc2 libjemalloc-dev make libssl-dev pkg-config
+
+RUN mkdir -p zero_bin
+COPY Cargo.toml .
+# Cleanup all workspace members and add selected crates again
+RUN sed -i '/members =/{:a;N;/]/!ba};//d' Cargo.toml
+RUN sed -i 's#\[workspace\]#\[workspace\]\nmembers = \["zero_bin\/worker", "zero_bin\/common", "zero_bin\/ops"\, "evm_arithmetization", "mpt_trie", "proc_macro"\]#' Cargo.toml
+COPY Cargo.lock .
+COPY ./rust-toolchain.toml ./
+
+COPY proof_gen proof_gen
+COPY mpt_trie mpt_trie
+COPY evm_arithmetization evm_arithmetization
+COPY proc_macro proc_macro
+COPY zero_bin/common zero_bin/common
+COPY zero_bin/ops zero_bin/ops
+COPY zero_bin/worker zero_bin/worker
+
+RUN \
+  touch zero_bin/common/src/lib.rs && \
+  touch zero_bin/ops/src/lib.rs && \
+  touch zero_bin/worker/src/main.rs && \
+  touch evm_arithmetization/src/lib.rs && \
+  touch mpt_trie/src/lib.rs && \
+  touch proc_macro/src/lib.rs
+
+# Disable the lld linker for now, as it's causing issues with the linkme package.
+# https://github.com/rust-lang/rust/pull/124129
+# https://github.com/dtolnay/linkme/pull/88
+ENV RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld'
+
+RUN cargo build --release --bin worker
+
+FROM debian:bullseye-slim
+RUN apt-get update && apt-get install -y ca-certificates libjemalloc2
+COPY --from=builder ./target/release/worker /usr/local/bin/worker
+ENTRYPOINT ["/usr/local/bin/worker"]
+