From 34ee9a1a46dcce6358e40f29c98a8057e8bd7aea Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Tue, 5 Nov 2024 11:24:53 +0100 Subject: [PATCH 1/9] Try to build no unit tests for the arm container. Signed-off-by: Johannes Kalmbach --- .github/workflows/docker-publish.yml | 3 ++- Dockerfile | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index c567f8e97b..9e655f2634 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -68,7 +68,8 @@ jobs: # future, then we can set this to `true`. Then the ARM64 image will also # be built on pull requests which allows for debugging without changing # the master branch. - if: false + #if: false + #if: true uses: docker/build-push-action@v6 with: context: . diff --git a/Dockerfile b/Dockerfile index bd514e1e12..7916924046 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y software-properties-common wget && add- RUN wget https://apt.kitware.com/kitware-archive.sh && chmod +x kitware-archive.sh &&./kitware-archive.sh FROM base as builder +ARG TARGETPLATFORM RUN apt-get update && apt-get install -y build-essential cmake libicu-dev tzdata pkg-config uuid-runtime uuid-dev git libjemalloc-dev ninja-build libzstd-dev libssl-dev libboost1.81-dev libboost-program-options1.81-dev libboost-iostreams1.81-dev libboost-url1.81-dev COPY . /app/ @@ -16,7 +17,9 @@ WORKDIR /app/ ENV DEBIAN_FRONTEND=noninteractive WORKDIR /app/build/ -RUN cmake -DCMAKE_BUILD_TYPE=Release -DLOGLEVEL=INFO -DUSE_PARALLEL=true -D_NO_TIMING_TESTS=ON -GNinja .. && ninja +RUN cmake -DCMAKE_BUILD_TYPE=Release -DLOGLEVEL=INFO -DUSE_PARALLEL=true -D_NO_TIMING_TESTS=ON -GNinja .. +RUN echo "Building for platform ->$TARGETPLATFORM<-" +RUN if ["$TARGETPLATFORM" = "linux/arm64"] ; then cmake --build . --target IndexBuilderMain ServerMain; else cmake --build . ; fi RUN ctest --rerun-failed --output-on-failure FROM base as runtime From e83b006d75f6f55c6e8abbde459d811a4da459e8 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Tue, 5 Nov 2024 11:25:43 +0100 Subject: [PATCH 2/9] Only build the ARM to check stuff... Signed-off-by: Johannes Kalmbach --- .github/workflows/docker-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9e655f2634..20cef2749c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -38,6 +38,7 @@ jobs: run: echo "pr_num=$(git log --format=%s -n 1 | sed -nr 's/.*\(\#([0-9]+)\)/\1/p')" >> $GITHUB_OUTPUT - name: Build X-86-64 uses: docker/build-push-action@v6 + if: false with: context: . # Docker multiplatform images require an entry in their manifests. If this @@ -56,6 +57,7 @@ jobs: tags: adfreiburg/qlever:test - name: E2E in Docker + if: false run: | sudo mkdir ${{github.workspace}}/e2e_data sudo chmod a+rwx ${{github.workspace}}/e2e_data From 2ba68e15f3cbe8853077f77e94a5d19a274591e9 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Tue, 5 Nov 2024 15:14:57 +0100 Subject: [PATCH 3/9] Update Dockerfile --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7916924046..bf1b030e94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,12 @@ FROM ubuntu:22.04 as base +ARG TARGETPLATFORM LABEL maintainer="Johannes Kalmbach " ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 ENV LC_CTYPE C.UTF-8 ENV DEBIAN_FRONTEND=noninteractive +RUN echo "Building for platform ->$TARGETPLATFORM<-" +RUN if [[ $TARGETPLATFORM -eq "linux/arm64"] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi RUN apt-get update && apt-get install -y software-properties-common wget && add-apt-repository -y ppa:mhier/libboost-latest RUN wget https://apt.kitware.com/kitware-archive.sh && chmod +x kitware-archive.sh &&./kitware-archive.sh @@ -19,7 +22,7 @@ ENV DEBIAN_FRONTEND=noninteractive WORKDIR /app/build/ RUN cmake -DCMAKE_BUILD_TYPE=Release -DLOGLEVEL=INFO -DUSE_PARALLEL=true -D_NO_TIMING_TESTS=ON -GNinja .. RUN echo "Building for platform ->$TARGETPLATFORM<-" -RUN if ["$TARGETPLATFORM" = "linux/arm64"] ; then cmake --build . --target IndexBuilderMain ServerMain; else cmake --build . ; fi +RUN if [[ $TARGETPLATFORM -eq "linux/arm64"] ; then cmake --build . --target IndexBuilderMain ServerMain; else cmake --build . ; fi RUN ctest --rerun-failed --output-on-failure FROM base as runtime From 555f3e0ce7004cccc959c538ddd71f6ba9c67fe8 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Tue, 5 Nov 2024 15:35:54 +0100 Subject: [PATCH 4/9] Update Dockerfile --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index bf1b030e94..bd09b1d0d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,14 +6,15 @@ ENV LC_ALL C.UTF-8 ENV LC_CTYPE C.UTF-8 ENV DEBIAN_FRONTEND=noninteractive RUN echo "Building for platform ->$TARGETPLATFORM<-" -RUN if [[ $TARGETPLATFORM -eq "linux/arm64"] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi +RUN if [[ $TARGETPLATFORM -eq "linux/arm64"]] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi RUN apt-get update && apt-get install -y software-properties-common wget && add-apt-repository -y ppa:mhier/libboost-latest RUN wget https://apt.kitware.com/kitware-archive.sh && chmod +x kitware-archive.sh &&./kitware-archive.sh FROM base as builder ARG TARGETPLATFORM +RUN if [[ $TARGETPLATFORM -eq "linux/arm64"]] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi RUN apt-get update && apt-get install -y build-essential cmake libicu-dev tzdata pkg-config uuid-runtime uuid-dev git libjemalloc-dev ninja-build libzstd-dev libssl-dev libboost1.81-dev libboost-program-options1.81-dev libboost-iostreams1.81-dev libboost-url1.81-dev - +RUN if [[ $TARGETPLATFORM -eq "linux/arm64"]] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi COPY . /app/ WORKDIR /app/ @@ -22,7 +23,7 @@ ENV DEBIAN_FRONTEND=noninteractive WORKDIR /app/build/ RUN cmake -DCMAKE_BUILD_TYPE=Release -DLOGLEVEL=INFO -DUSE_PARALLEL=true -D_NO_TIMING_TESTS=ON -GNinja .. RUN echo "Building for platform ->$TARGETPLATFORM<-" -RUN if [[ $TARGETPLATFORM -eq "linux/arm64"] ; then cmake --build . --target IndexBuilderMain ServerMain; else cmake --build . ; fi +RUN if [[ $TARGETPLATFORM -eq "linux/arm64"]] ; then cmake --build . --target IndexBuilderMain ServerMain; else cmake --build . ; fi RUN ctest --rerun-failed --output-on-failure FROM base as runtime From b8dd3a030665c312ba8eb4708fc1980ac50cf64b Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Tue, 5 Nov 2024 16:37:24 +0100 Subject: [PATCH 5/9] Update Dockerfile --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index bd09b1d0d3..874b2aad7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,15 +6,15 @@ ENV LC_ALL C.UTF-8 ENV LC_CTYPE C.UTF-8 ENV DEBIAN_FRONTEND=noninteractive RUN echo "Building for platform ->$TARGETPLATFORM<-" -RUN if [[ $TARGETPLATFORM -eq "linux/arm64"]] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi +RUN if [ $TARGETPLATFORM = "linux/arm64"] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi RUN apt-get update && apt-get install -y software-properties-common wget && add-apt-repository -y ppa:mhier/libboost-latest RUN wget https://apt.kitware.com/kitware-archive.sh && chmod +x kitware-archive.sh &&./kitware-archive.sh FROM base as builder ARG TARGETPLATFORM -RUN if [[ $TARGETPLATFORM -eq "linux/arm64"]] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi +RUN if [ $TARGETPLATFORM = "linux/arm64"] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi RUN apt-get update && apt-get install -y build-essential cmake libicu-dev tzdata pkg-config uuid-runtime uuid-dev git libjemalloc-dev ninja-build libzstd-dev libssl-dev libboost1.81-dev libboost-program-options1.81-dev libboost-iostreams1.81-dev libboost-url1.81-dev -RUN if [[ $TARGETPLATFORM -eq "linux/arm64"]] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi +RUN if [ $TARGETPLATFORM = "linux/arm64"] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi COPY . /app/ WORKDIR /app/ @@ -23,7 +23,7 @@ ENV DEBIAN_FRONTEND=noninteractive WORKDIR /app/build/ RUN cmake -DCMAKE_BUILD_TYPE=Release -DLOGLEVEL=INFO -DUSE_PARALLEL=true -D_NO_TIMING_TESTS=ON -GNinja .. RUN echo "Building for platform ->$TARGETPLATFORM<-" -RUN if [[ $TARGETPLATFORM -eq "linux/arm64"]] ; then cmake --build . --target IndexBuilderMain ServerMain; else cmake --build . ; fi +RUN if [ $TARGETPLATFORM = "linux/arm64"] ; then cmake --build . --target IndexBuilderMain ServerMain; else cmake --build . ; fi RUN ctest --rerun-failed --output-on-failure FROM base as runtime From f3a345e1f5e79d6a000088a61dfdadf12d8976bc Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Tue, 5 Nov 2024 16:39:04 +0100 Subject: [PATCH 6/9] Update Dockerfile --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 874b2aad7c..07bef5177e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,15 +6,15 @@ ENV LC_ALL C.UTF-8 ENV LC_CTYPE C.UTF-8 ENV DEBIAN_FRONTEND=noninteractive RUN echo "Building for platform ->$TARGETPLATFORM<-" -RUN if [ $TARGETPLATFORM = "linux/arm64"] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi +RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi RUN apt-get update && apt-get install -y software-properties-common wget && add-apt-repository -y ppa:mhier/libboost-latest RUN wget https://apt.kitware.com/kitware-archive.sh && chmod +x kitware-archive.sh &&./kitware-archive.sh FROM base as builder ARG TARGETPLATFORM -RUN if [ $TARGETPLATFORM = "linux/arm64"] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi +RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi RUN apt-get update && apt-get install -y build-essential cmake libicu-dev tzdata pkg-config uuid-runtime uuid-dev git libjemalloc-dev ninja-build libzstd-dev libssl-dev libboost1.81-dev libboost-program-options1.81-dev libboost-iostreams1.81-dev libboost-url1.81-dev -RUN if [ $TARGETPLATFORM = "linux/arm64"] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi +RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi COPY . /app/ WORKDIR /app/ @@ -23,7 +23,7 @@ ENV DEBIAN_FRONTEND=noninteractive WORKDIR /app/build/ RUN cmake -DCMAKE_BUILD_TYPE=Release -DLOGLEVEL=INFO -DUSE_PARALLEL=true -D_NO_TIMING_TESTS=ON -GNinja .. RUN echo "Building for platform ->$TARGETPLATFORM<-" -RUN if [ $TARGETPLATFORM = "linux/arm64"] ; then cmake --build . --target IndexBuilderMain ServerMain; else cmake --build . ; fi +RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then cmake --build . --target IndexBuilderMain ServerMain; else cmake --build . ; fi RUN ctest --rerun-failed --output-on-failure FROM base as runtime From 377a9c3d1defbc1dfe38d8303d7e65033e127094 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Wed, 6 Nov 2024 08:45:24 +0100 Subject: [PATCH 7/9] Find out if it works like this Signed-off-by: Johannes Kalmbach --- Dockerfile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 07bef5177e..959cc0ba7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,15 @@ FROM ubuntu:22.04 as base -ARG TARGETPLATFORM LABEL maintainer="Johannes Kalmbach " ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 ENV LC_CTYPE C.UTF-8 ENV DEBIAN_FRONTEND=noninteractive -RUN echo "Building for platform ->$TARGETPLATFORM<-" -RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi RUN apt-get update && apt-get install -y software-properties-common wget && add-apt-repository -y ppa:mhier/libboost-latest RUN wget https://apt.kitware.com/kitware-archive.sh && chmod +x kitware-archive.sh &&./kitware-archive.sh FROM base as builder ARG TARGETPLATFORM -RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi RUN apt-get update && apt-get install -y build-essential cmake libicu-dev tzdata pkg-config uuid-runtime uuid-dev git libjemalloc-dev ninja-build libzstd-dev libssl-dev libboost1.81-dev libboost-program-options1.81-dev libboost-iostreams1.81-dev libboost-url1.81-dev -RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then echo "target is ARM"; else echo "target is not ARM, probably AMD64"; fi COPY . /app/ WORKDIR /app/ @@ -22,9 +17,12 @@ ENV DEBIAN_FRONTEND=noninteractive WORKDIR /app/build/ RUN cmake -DCMAKE_BUILD_TYPE=Release -DLOGLEVEL=INFO -DUSE_PARALLEL=true -D_NO_TIMING_TESTS=ON -GNinja .. -RUN echo "Building for platform ->$TARGETPLATFORM<-" +# When cross-compiling the container for ARM64, then compiling and running all tests runs into a timeout on GitHub actions, +# so we disable tests for this platform. +# TODO(joka921) reenable these tests as soon as we can use a native ARM64 platform to compile the docker container. +RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then echo "target is ARM64, don't build tests to avoid timeout"; fi RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then cmake --build . --target IndexBuilderMain ServerMain; else cmake --build . ; fi -RUN ctest --rerun-failed --output-on-failure +RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then ctest --rerun-failed --output-on-failure ; fi FROM base as runtime WORKDIR /app From 0854355771e415de9b8dc31f698c9686b862307b Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Wed, 6 Nov 2024 16:34:17 +0100 Subject: [PATCH 8/9] The hopefully final fix. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 959cc0ba7b..15d7754191 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,10 +19,10 @@ WORKDIR /app/build/ RUN cmake -DCMAKE_BUILD_TYPE=Release -DLOGLEVEL=INFO -DUSE_PARALLEL=true -D_NO_TIMING_TESTS=ON -GNinja .. # When cross-compiling the container for ARM64, then compiling and running all tests runs into a timeout on GitHub actions, # so we disable tests for this platform. -# TODO(joka921) reenable these tests as soon as we can use a native ARM64 platform to compile the docker container. +# TODO(joka921) re-enable these tests as soon as we can use a native ARM64 platform to compile the docker container. RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then echo "target is ARM64, don't build tests to avoid timeout"; fi RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then cmake --build . --target IndexBuilderMain ServerMain; else cmake --build . ; fi -RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then ctest --rerun-failed --output-on-failure ; fi +RUN if [ $TARGETPLATFORM = "linux/arm64" ] ; then echo "Skipping tests for ARM64" ; else ctest --rerun-failed --output-on-failure ; fi FROM base as runtime WORKDIR /app From ba8195b83387bf9e2f4ddea109bf5ad266397110 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 7 Nov 2024 11:46:29 +0100 Subject: [PATCH 9/9] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 20cef2749c..c567f8e97b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -38,7 +38,6 @@ jobs: run: echo "pr_num=$(git log --format=%s -n 1 | sed -nr 's/.*\(\#([0-9]+)\)/\1/p')" >> $GITHUB_OUTPUT - name: Build X-86-64 uses: docker/build-push-action@v6 - if: false with: context: . # Docker multiplatform images require an entry in their manifests. If this @@ -57,7 +56,6 @@ jobs: tags: adfreiburg/qlever:test - name: E2E in Docker - if: false run: | sudo mkdir ${{github.workspace}}/e2e_data sudo chmod a+rwx ${{github.workspace}}/e2e_data @@ -70,8 +68,7 @@ jobs: # future, then we can set this to `true`. Then the ARM64 image will also # be built on pull requests which allows for debugging without changing # the master branch. - #if: false - #if: true + if: false uses: docker/build-push-action@v6 with: context: .