From cc43c4651c58b4f9a14315b1f7dbf9816a1c1fc5 Mon Sep 17 00:00:00 2001 From: G-D-Petrov Date: Wed, 23 Oct 2024 10:07:15 +0300 Subject: [PATCH 1/6] Test new docker img --- .github/workflows/build.yml | 22 +++++++++++----------- cpp/CMakeLists.txt | 4 +++- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7097864ad..52c4a5432d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,15 +24,15 @@ concurrency: group: ${{github.ref}} cancel-in-progress: true jobs: - cibw_docker_image: - uses: ./.github/workflows/cibw_docker_image.yml - permissions: {packages: write} - with: - cibuildwheel_ver: "2.12.1" - force_update: false + # cibw_docker_image: + # uses: ./.github/workflows/cibw_docker_image.yml + # permissions: {packages: write} + # with: + # cibuildwheel_ver: "2.12.1" + # force_update: false common_config: - needs: [cibw_docker_image] + # needs: [cibw_docker_image] runs-on: ubuntu-22.04 strategy: matrix: @@ -52,7 +52,7 @@ jobs: mongodb: image: "mongo:4.4" container: - image: ${{needs.cibw_docker_image.outputs.tag}} + image: ghcr.io/man-group/arcticdb-container:20241022-f5ffd9b volumes: - /:/mnt windows_matrix: @@ -79,7 +79,7 @@ jobs: env: DEFAULT_PRESET: ${{startsWith(github.repository, 'man-group/ArcticDB') && 'release' || vars.DEFAULT_CMAKE_PRESET_TYPE || 'debug'}} outputs: - cibuildwheel_ver: ${{needs.cibw_docker_image.outputs.cibuildwheel_ver}} + cibuildwheel_ver: "2.12.1" publish_env: ${{inputs.publish_env || env.PUBLISH_ENV || ''}} pypi_publish: ${{inputs.pypi_publish || env.PYPI_PUBLISH}} cmake_preset_type_resolved: ${{inputs.cmake_preset_type != '-' && inputs.cmake_preset_type || env.CMAKE_PRESET_TYPE}} @@ -181,7 +181,7 @@ jobs: job_type: build-python-wheels python3: ${{matrix.python3}} cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}} - cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}} + cibw_version: "2.12.1" cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} matrix: ${{toJson(matrix.matrix_override)}} python_deps_ids: ${{toJson(matrix.python_deps_ids)}} @@ -211,7 +211,7 @@ jobs: with: job_type: build-python-wheels python3: ${{matrix.python3}} - cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}} + cibw_version: "2.12.1" cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} matrix: ${{needs.common_config.outputs.windows_matrix}} persistent_storage: ${{ inputs.persistent_storage }} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 03802012b1..b3e3662e59 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -81,7 +81,9 @@ else() if(${ARCTICDB_USING_CONDA}) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra") else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra") + # DO NOT MERGE THIS + # This is just temprorary until we fix all warnings + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra") endif() set(CMAKE_CXX_FLAGS_DEBUG "-ggdb") if(DEFINED ENV{ARCTICDB_CODE_COVERAGE_BUILD}) From 8d15e33a6d99a80fff7bffa4364c9079b4c01e68 Mon Sep 17 00:00:00 2001 From: G-D-Petrov Date: Wed, 23 Oct 2024 10:09:24 +0300 Subject: [PATCH 2/6] Fix image tag --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 52c4a5432d..8fd3d691ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -139,19 +139,19 @@ jobs: python_deps_ids: ${{toJson(matrix.python_deps_ids)}} cpp-test-linux: - needs: [cibw_docker_image, common_config] + needs: [common_config] name: Linux C++ Tests uses: ./.github/workflows/build_steps.yml secrets: inherit with: job_type: cpp-tests cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} - cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}} + cibw_image_tag: ghcr.io/man-group/arcticdb-container:20241022-f5ffd9b matrix: ${{needs.common_config.outputs.linux_matrix}} build-python-wheels-linux: # Then use the cached compilation artifacts to build other python versions concurrently in cibuildwheels - needs: [cibw_docker_image, common_config] + needs: [common_config] strategy: fail-fast: false matrix: @@ -180,7 +180,7 @@ jobs: with: job_type: build-python-wheels python3: ${{matrix.python3}} - cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}} + cibw_image_tag: ghcr.io/man-group/arcticdb-container:20241022-f5ffd9b cibw_version: "2.12.1" cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} matrix: ${{toJson(matrix.matrix_override)}} From 3703dd0d54972e5522c206e02f25e09bf6a83b23 Mon Sep 17 00:00:00 2001 From: G-D-Petrov Date: Wed, 23 Oct 2024 10:16:05 +0300 Subject: [PATCH 3/6] Add dockerfile publish --- .github/workflows/build_container.yml | 34 +++++++++++++++++++++++++++ docker/Dockerfile | 25 ++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 .github/workflows/build_container.yml create mode 100644 docker/Dockerfile diff --git a/.github/workflows/build_container.yml b/.github/workflows/build_container.yml new file mode 100644 index 0000000000..f662134189 --- /dev/null +++ b/.github/workflows/build_container.yml @@ -0,0 +1,34 @@ +name: Docker Container Build +on: + workflow_dispatch: + push: + branches: + - master + +jobs: + build: + permissions: + contents: read + packages: write + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3.3.0 + + - name: Choose image tag + run: | + sha=$(git rev-parse --short HEAD) + image_ver="$(date '+%Y%m%d')-${sha}" + image_name="ghcr.io/man-group/arcticdb-container" + echo -e "image_ver=$image_ver\nimage_name=$image_name\noutput_tag=$image_name:$image_ver" | tee -a $GITHUB_ENV + + - name: Build Docker image + run: | + docker build -t $image_name . -f docker/Dockerfile + + - name: Publish Docker image to GHCR + # if: startsWith(github.ref, 'refs/tags') || github.ref == 'refs/heads/master' + run: | + docker login ghcr.io -u token -p "${{secrets.GITHUB_TOKEN}}" + docker tag $image_name $image_name:$image_ver + docker push $image_name:$image_ver \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000..5d2dc8c06f --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,25 @@ +FROM quay.io/pypa/manylinux_2_28_x86_64 + +RUN dnf update -y +RUN dnf install -y python3.11 python3.11-devel python3.11-pip curl wget zip unzip tar perl-IPC-Cmd \ +flex krb5-devel cyrus-sasl-devel epel-release libcurl-devel + +RUN rpm --import https://download.mono-project.com/repo/xamarin.gpg +RUN dnf config-manager --add-repo https://download.mono-project.com/repo/centos8-stable.repo +RUN dnf install -y mono-complete + +RUN dnf clean all + +RUN export CMAKE_C_COMPILER=/opt/rh/gcc-toolset-12/root/bin/gcc +RUN export CMAKE_CXX_COMPILER=/opt/rh/gcc-toolset-12/root/bin/g++ +RUN export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:${LD_LIBRARY_PATH} +RUN export PATH=/opt/rh/gcc-toolset-12/root/usr/bin:${PATH} + +RUN wget -nv https://github.com/mozilla/sccache/releases/download/v0.8.2/sccache-v0.8.2-x86_64-unknown-linux-musl.tar.gz +RUN tar xvf sccache*.tar.gz +RUN mv sccache-*/sccache . +RUN chmod 555 sccache + +RUN cp sccache /usr/local/bin/ + +WORKDIR /root From d138cfc6376d8bba76ba41571bef51c301ed7828 Mon Sep 17 00:00:00 2001 From: G-D-Petrov Date: Wed, 23 Oct 2024 10:18:33 +0300 Subject: [PATCH 4/6] Change trigger --- .github/workflows/build_container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_container.yml b/.github/workflows/build_container.yml index f662134189..e8f01ac3ad 100644 --- a/.github/workflows/build_container.yml +++ b/.github/workflows/build_container.yml @@ -1,7 +1,7 @@ name: Docker Container Build on: workflow_dispatch: - push: + pull_request: branches: - master From 6a759d6a34917a62f7372ad0611ba855ab1f425d Mon Sep 17 00:00:00 2001 From: G-D-Petrov Date: Wed, 23 Oct 2024 10:34:07 +0300 Subject: [PATCH 5/6] try with the old build docker step --- .github/workflows/build.yml | 30 ++++++++++++------------- build_tooling/build_many_linux_image.sh | 29 +++++++++++++++--------- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8fd3d691ad..e7097864ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,15 +24,15 @@ concurrency: group: ${{github.ref}} cancel-in-progress: true jobs: - # cibw_docker_image: - # uses: ./.github/workflows/cibw_docker_image.yml - # permissions: {packages: write} - # with: - # cibuildwheel_ver: "2.12.1" - # force_update: false + cibw_docker_image: + uses: ./.github/workflows/cibw_docker_image.yml + permissions: {packages: write} + with: + cibuildwheel_ver: "2.12.1" + force_update: false common_config: - # needs: [cibw_docker_image] + needs: [cibw_docker_image] runs-on: ubuntu-22.04 strategy: matrix: @@ -52,7 +52,7 @@ jobs: mongodb: image: "mongo:4.4" container: - image: ghcr.io/man-group/arcticdb-container:20241022-f5ffd9b + image: ${{needs.cibw_docker_image.outputs.tag}} volumes: - /:/mnt windows_matrix: @@ -79,7 +79,7 @@ jobs: env: DEFAULT_PRESET: ${{startsWith(github.repository, 'man-group/ArcticDB') && 'release' || vars.DEFAULT_CMAKE_PRESET_TYPE || 'debug'}} outputs: - cibuildwheel_ver: "2.12.1" + cibuildwheel_ver: ${{needs.cibw_docker_image.outputs.cibuildwheel_ver}} publish_env: ${{inputs.publish_env || env.PUBLISH_ENV || ''}} pypi_publish: ${{inputs.pypi_publish || env.PYPI_PUBLISH}} cmake_preset_type_resolved: ${{inputs.cmake_preset_type != '-' && inputs.cmake_preset_type || env.CMAKE_PRESET_TYPE}} @@ -139,19 +139,19 @@ jobs: python_deps_ids: ${{toJson(matrix.python_deps_ids)}} cpp-test-linux: - needs: [common_config] + needs: [cibw_docker_image, common_config] name: Linux C++ Tests uses: ./.github/workflows/build_steps.yml secrets: inherit with: job_type: cpp-tests cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} - cibw_image_tag: ghcr.io/man-group/arcticdb-container:20241022-f5ffd9b + cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}} matrix: ${{needs.common_config.outputs.linux_matrix}} build-python-wheels-linux: # Then use the cached compilation artifacts to build other python versions concurrently in cibuildwheels - needs: [common_config] + needs: [cibw_docker_image, common_config] strategy: fail-fast: false matrix: @@ -180,8 +180,8 @@ jobs: with: job_type: build-python-wheels python3: ${{matrix.python3}} - cibw_image_tag: ghcr.io/man-group/arcticdb-container:20241022-f5ffd9b - cibw_version: "2.12.1" + cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}} + cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}} cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} matrix: ${{toJson(matrix.matrix_override)}} python_deps_ids: ${{toJson(matrix.python_deps_ids)}} @@ -211,7 +211,7 @@ jobs: with: job_type: build-python-wheels python3: ${{matrix.python3}} - cibw_version: "2.12.1" + cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}} cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} matrix: ${{needs.common_config.outputs.windows_matrix}} persistent_storage: ${{ inputs.persistent_storage }} diff --git a/build_tooling/build_many_linux_image.sh b/build_tooling/build_many_linux_image.sh index ed5ee61883..dba3f162c6 100755 --- a/build_tooling/build_many_linux_image.sh +++ b/build_tooling/build_many_linux_image.sh @@ -7,7 +7,7 @@ if [[ -z "$manylinux_image" ]] ; then v${cibuildwheel_ver:?'Must set either manylinux_image or cibuildwheel_ver environment variable'} url="https://github.com/pypa/cibuildwheel/raw/v${cibuildwheel_ver}/cibuildwheel/resources/pinned_docker_images.cfg" - manylinux_image=$(curl -sL "$url" | awk "/${image_grep:-manylinux2014_x86_64}/ { print \$3 ; exit }" ) + manylinux_image=$(curl -sL "$url" | awk "/${image_grep:-manylinux_2_28_x86_64}/ { print \$3 ; exit }" ) if [[ -z "$manylinux_image" ]] ; then echo "Failed to parse source image from cibuildwheel repo: ${url}" >&2 exit 1 @@ -37,17 +37,24 @@ chmod 555 sccache echo " FROM $manylinux_image -RUN rpmkeys --import 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF' && \ - curl https://download.mono-project.com/repo/centos7-stable.repo > /etc/yum.repos.d/mono-centos7-stable.repo && \ - sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://olcentgbl.trafficmanager.net|' /etc/yum.repos.d/CentOS-Base.repo && \ - sed -ir 's/socket_timeout=3/socket_timeout=1/ ; s/maxhostfileage.*/maxhostfileage=1/ ;\ - s/#?exclude.*/exclude=.edu/' /etc/yum/pluginconf.d/fastestmirror.conf + +RUN dnf update -y +RUN dnf install -y python3.11 python3.11-devel python3.11-pip curl wget zip unzip tar perl-IPC-Cmd \ +flex krb5-devel cyrus-sasl-devel epel-release libcurl-devel + +RUN rpm --import https://download.mono-project.com/repo/xamarin.gpg +RUN dnf config-manager --add-repo https://download.mono-project.com/repo/centos8-stable.repo +RUN dnf install -y mono-complete + +RUN dnf clean all + +RUN export CMAKE_C_COMPILER=/opt/rh/gcc-toolset-12/root/bin/gcc +RUN export CMAKE_CXX_COMPILER=/opt/rh/gcc-toolset-12/root/bin/g++ +RUN export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:${LD_LIBRARY_PATH} +RUN export PATH=/opt/rh/gcc-toolset-12/root/usr/bin:${PATH} + ADD sccache /usr/local/bin/ -RUN yum update -y && \ - yum install -y zip jq less devtoolset-11-gdb perl-IPC-Cmd \ - openssl-devel cyrus-sasl-devel devtoolset-10-libatomic-devel libcurl-devel python3-devel flex && \ - rpm -Uvh --nodeps \$(repoquery --location mono-{core,web,devel,data,wcf,winfx}) && \ - yum clean all && touch /etc/arcticdb_deps_installed + LABEL io.arcticdb.cibw_ver=\"${cibuildwheel_ver}\" io.arcticdb.base=\"${manylinux_image}\" " > Dockerfile From cb00b2bbb1bcff031a9baed098d32e030a38cf22 Mon Sep 17 00:00:00 2001 From: G-D-Petrov Date: Wed, 23 Oct 2024 10:39:19 +0300 Subject: [PATCH 6/6] Test with newer cibw version --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7097864ad..7a7f3e3847 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,7 +28,7 @@ jobs: uses: ./.github/workflows/cibw_docker_image.yml permissions: {packages: write} with: - cibuildwheel_ver: "2.12.1" + cibuildwheel_ver: "2.21.3" force_update: false common_config: