Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test new docker img #1938

Draft
wants to merge 6 commits into
base: master
Choose a base branch
from
Draft
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
try with the old build docker step
G-D-Petrov committed Oct 23, 2024
commit 6a759d6a34917a62f7372ad0611ba855ab1f425d
30 changes: 15 additions & 15 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
@@ -24,15 +24,15 @@ concurrency:
group: ${{github.ref}}
cancel-in-progress: true
jobs:
# cibw_docker_image:
# uses: ./.github/workflows/cibw_docker_image.yml
# permissions: {packages: write}
# with:
# cibuildwheel_ver: "2.12.1"
# force_update: false
cibw_docker_image:
uses: ./.github/workflows/cibw_docker_image.yml
permissions: {packages: write}
with:
cibuildwheel_ver: "2.12.1"
force_update: false

common_config:
# needs: [cibw_docker_image]
needs: [cibw_docker_image]
runs-on: ubuntu-22.04
strategy:
matrix:
@@ -52,7 +52,7 @@ jobs:
mongodb:
image: "mongo:4.4"
container:
image: ghcr.io/man-group/arcticdb-container:20241022-f5ffd9b
image: ${{needs.cibw_docker_image.outputs.tag}}
volumes:
- /:/mnt
windows_matrix:
@@ -79,7 +79,7 @@ jobs:
env:
DEFAULT_PRESET: ${{startsWith(github.repository, 'man-group/ArcticDB') && 'release' || vars.DEFAULT_CMAKE_PRESET_TYPE || 'debug'}}
outputs:
cibuildwheel_ver: "2.12.1"
cibuildwheel_ver: ${{needs.cibw_docker_image.outputs.cibuildwheel_ver}}
publish_env: ${{inputs.publish_env || env.PUBLISH_ENV || ''}}
pypi_publish: ${{inputs.pypi_publish || env.PYPI_PUBLISH}}
cmake_preset_type_resolved: ${{inputs.cmake_preset_type != '-' && inputs.cmake_preset_type || env.CMAKE_PRESET_TYPE}}
@@ -139,19 +139,19 @@ jobs:
python_deps_ids: ${{toJson(matrix.python_deps_ids)}}

cpp-test-linux:
needs: [common_config]
needs: [cibw_docker_image, common_config]
name: Linux C++ Tests
uses: ./.github/workflows/build_steps.yml
secrets: inherit
with:
job_type: cpp-tests
cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
cibw_image_tag: ghcr.io/man-group/arcticdb-container:20241022-f5ffd9b
cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}}
matrix: ${{needs.common_config.outputs.linux_matrix}}

build-python-wheels-linux:
# Then use the cached compilation artifacts to build other python versions concurrently in cibuildwheels
needs: [common_config]
needs: [cibw_docker_image, common_config]
strategy:
fail-fast: false
matrix:
@@ -180,8 +180,8 @@ jobs:
with:
job_type: build-python-wheels
python3: ${{matrix.python3}}
cibw_image_tag: ghcr.io/man-group/arcticdb-container:20241022-f5ffd9b
cibw_version: "2.12.1"
cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}}
cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}}
cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
matrix: ${{toJson(matrix.matrix_override)}}
python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
@@ -211,7 +211,7 @@ jobs:
with:
job_type: build-python-wheels
python3: ${{matrix.python3}}
cibw_version: "2.12.1"
cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}}
cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
matrix: ${{needs.common_config.outputs.windows_matrix}}
persistent_storage: ${{ inputs.persistent_storage }}
29 changes: 18 additions & 11 deletions build_tooling/build_many_linux_image.sh
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@ if [[ -z "$manylinux_image" ]] ; then
v${cibuildwheel_ver:?'Must set either manylinux_image or cibuildwheel_ver environment variable'}

url="https://github.com/pypa/cibuildwheel/raw/v${cibuildwheel_ver}/cibuildwheel/resources/pinned_docker_images.cfg"
manylinux_image=$(curl -sL "$url" | awk "/${image_grep:-manylinux2014_x86_64}/ { print \$3 ; exit }" )
manylinux_image=$(curl -sL "$url" | awk "/${image_grep:-manylinux_2_28_x86_64}/ { print \$3 ; exit }" )
if [[ -z "$manylinux_image" ]] ; then
echo "Failed to parse source image from cibuildwheel repo: ${url}" >&2
exit 1
@@ -37,17 +37,24 @@ chmod 555 sccache

echo "
FROM $manylinux_image
RUN rpmkeys --import 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF' && \
curl https://download.mono-project.com/repo/centos7-stable.repo > /etc/yum.repos.d/mono-centos7-stable.repo && \
sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://olcentgbl.trafficmanager.net|' /etc/yum.repos.d/CentOS-Base.repo && \
sed -ir 's/socket_timeout=3/socket_timeout=1/ ; s/maxhostfileage.*/maxhostfileage=1/ ;\
s/#?exclude.*/exclude=.edu/' /etc/yum/pluginconf.d/fastestmirror.conf

RUN dnf update -y
RUN dnf install -y python3.11 python3.11-devel python3.11-pip curl wget zip unzip tar perl-IPC-Cmd \
flex krb5-devel cyrus-sasl-devel epel-release libcurl-devel

RUN rpm --import https://download.mono-project.com/repo/xamarin.gpg
RUN dnf config-manager --add-repo https://download.mono-project.com/repo/centos8-stable.repo
RUN dnf install -y mono-complete

RUN dnf clean all

RUN export CMAKE_C_COMPILER=/opt/rh/gcc-toolset-12/root/bin/gcc
RUN export CMAKE_CXX_COMPILER=/opt/rh/gcc-toolset-12/root/bin/g++
RUN export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:${LD_LIBRARY_PATH}
RUN export PATH=/opt/rh/gcc-toolset-12/root/usr/bin:${PATH}

ADD sccache /usr/local/bin/
RUN yum update -y && \
yum install -y zip jq less devtoolset-11-gdb perl-IPC-Cmd \
openssl-devel cyrus-sasl-devel devtoolset-10-libatomic-devel libcurl-devel python3-devel flex && \
rpm -Uvh --nodeps \$(repoquery --location mono-{core,web,devel,data,wcf,winfx}) && \
yum clean all && touch /etc/arcticdb_deps_installed

LABEL io.arcticdb.cibw_ver=\"${cibuildwheel_ver}\" io.arcticdb.base=\"${manylinux_image}\"
" > Dockerfile