Refactor linux build images

The linux build Dockerfiles have grown into a tangled mess of
bootstrapping a bunch of software in a single `RUN` instruction.  Worse
yet, the script is slightly different in every target and maintenance is
just not great.

Fix this finally by splitting out toolchain bootstrap into nice shell
scripts and inject them into Dockerfiles the same way we inject the
entrypoint script.

Makefile

@@ -13,7 +13,7 @@ SUPPORTED_TARGETS = \
 	centos-7 \
 	centos-8 \
 	rockylinux-9 \
-	fedora-29 \
+	fedora-40 \
 	linux-x86_64 \
 	linuxmusl-x86_64 \
 	linux-aarch64 \

integration/linux/build/Dockerfile-centos.template

@@ -1,309 +1,84 @@
-FROM centos:%%PLACEHOLDER%%
+ARG DOCKER_ARCH=
+FROM ${DOCKER_ARCH}centos:%%PLACEHOLDER%%
 
-# ensure local python is preferred over distribution python
-ENV PATH /usr/local/bin:/usr/local/cargo/bin:/usr/local/go/bin:$PATH
-
-# http://bugs.python.org/issue19846
-# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
-ENV LANG en_US.UTF-8
+ENTRYPOINT ["/entrypoint.sh"]
 
+ENV PATH /usr/local/bin:/usr/local/cargo/bin:/usr/local/go/bin:$PATH
 ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib"
 
+ENV LANG C
+
+ENV RUSTUP_HOME /usr/local/rustup
+ENV CARGO_HOME /usr/local/cargo
 %%IF VARIANT=7%%
-ENV GCC_VERSION 11
+ENV OPENSSL_VERSION 1.1.1w
+ENV PYTHON_LOCAL_OPENSSL 1
 %%ENDIF%%
 
 # rpm on centos 7 iterates over all fds up to the limit, which is
 # extremely slow. Force it to be small with ulimit.
 #
 # We need to disable the mirrorlist and explicitly set up a
 # baseurl. Switch to the archive.kernel.org mirror while we are at it.
-RUN set -ex \
-	&& ulimit -n 1024 \
-	&& ( [ -e /etc/yum/pluginconf.d/fastestmirror.conf ] && sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf || true ) \
+RUN ulimit -n 1024 \
+	&& echo precedence ::ffff:0:0/96 100 >> /etc/gai.conf \
     && sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* \
     && sed -i 's|# \?baseurl=http://mirror.centos.org|baseurl=https://archive.kernel.org/centos-vault/|g' /etc/yum.repos.d/CentOS-* \
     && yum update -y \
-%%IF VARIANT=7%%
-    && yum install -y centos-release-scl epel-release \
-    && sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-SCLo* \
-    && sed -i "s|# \?baseurl=http://mirror.centos.org|baseurl=https://archive.kernel.org/centos-vault/|g" /etc/yum.repos.d/CentOS-SCLo* \
-    && yum update -y \
-%%ENDIF%%
-	&& echo "repo setup done"
-
-# rpm on centos 7 iterates over all fds up to the limit, which is
-# extremely slow. Force it to be small.
-RUN ulimit -n 1024 \
-    && yum install -y \
-    wget gcc gcc-c++ make patch zlib-devel sqlite-devel bzip2-devel \
-	openssl-devel \
-%%IF VARIANT=7%%
-	openssl11-devel \
-	devtoolset-${GCC_VERSION} \
-%%ENDIF%%
-    expat-devel ncurses-devel gdbm-devel readline-devel libuuid-devel \
-	curl-devel xz-devel libffi-devel gettext glibc-langpack-en \
-    perl-Getopt-Long perl-IPC-Cmd perl-Data-Dumper \
-	sudo ca-certificates cmake tcl file zip unzip bzip2
+	&& yum install -y \
+		bison \
+		bzip2 \
+		bzip2-devel \
+		ca-certificates \
+		curl-devel \
+		expat-devel \
+		file \
+		flex \
+		gcc \
+		gcc-c++ \
+		gdbm-devel \
+		gettext \
+		libffi-devel \
+		libuuid-devel \
+		make \
+		ncurses-devel \
+		openssl-devel \
+		patch \
+		perl-Data-Dumper \
+		perl-Getopt-Long \
+		perl-IPC-Cmd \
+		readline-devel \
+		sqlite-devel \
+		sudo \
+		tcl \
+		unzip \
+		wget \
+		xz \
+		xz-devel \
+		zip \
+		zlib-devel \
+		zstd
 
 %%IF VARIANT=7%%
-RUN printf "#!/bin/bash\nsource scl_source enable devtoolset-${GCC_VERSION}" \
-	> /etc/profile.d/enabledevtoolset${GCC_VERSION}.sh
+# aarch64 images somehow don't have the en_US locale
+RUN localedef -f UTF-8 -i en_US en_US.UTF-8
 %%ENDIF%%
 
-# Needed to pick up SCL stuff from profile.d/
-SHELL ["/bin/bash", "--login", "-c"]
-
-ENV GPG_KEY %%PLACEHOLDER%%
-ENV PYTHON_VERSION %%PLACEHOLDER%%
-ENV GIT_VERSION 2.20.1
-ENV TAR_VERSION latest
-ENV RUSTUP_HOME /usr/local/rustup
-ENV CARGO_HOME /usr/local/cargo
-ENV GO_VERSION %%PLACEHOLDER%%
-ENV PATH /usr/local/cargo/bin:$PATH
-ENV RUST_VERSION %%PLACEHOLDER%%
-ENV NODE_VERSION %%PLACEHOLDER%%
-ENV YARN_VERSION %%PLACEHOLDER%%
-ENV CMAKE_VERSION %%PLACEHOLDER%%
-ENV GO_VERSION %%PLACEHOLDER%%
-ENV NINJA_VERSION 1.12.1
-ENV PKGCONF_VERSION 2.3.0
+%%SCRIPTS%%
 
 RUN set -ex \
-	\
-	&& curl -fsSLo python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
-	&& curl -fsSLo python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
-	&& curl -fsSLo git.tar.xz "https://mirrors.edge.kernel.org/pub/software/scm/git/git-${GIT_VERSION}.tar.xz" \
-	&& curl -fsSLo git.tar.sign "https://mirrors.edge.kernel.org/pub/software/scm/git/git-${GIT_VERSION}.tar.sign" \
-	&& curl -fsSLo tar.tar.xz "https://alpha.gnu.org/gnu/tar/tar-${TAR_VERSION}.tar.xz" \
-	&& curl -fsSLo tar.tar.xz.sign "https://alpha.gnu.org/gnu/tar/tar-${TAR_VERSION}.tar.xz.sig" \
-	&& curl -fsSLo pkgconf.tar.xz "https://distfiles.ariadne.space/pkgconf/pkgconf-${PKGCONF_VERSION}.tar.xz" \
-	&& export GNUPGHOME="$(mktemp -d)" \
-	&& echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \
-    && for key in \
-      "${PYTHON_KEYS[@]}"
-    ; do \
-      gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$key" || \
-      gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" ; \
-    done \
-    && for key in \
-      "${NODE_KEYS[@]}"
-    ; do \
-      gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$key" || \
-      gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" ; \
-    done \
-    && for key in \
-      "${YARN_KEYS[@]}"
-    ; do \
-      gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$key" || \
-      gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" ; \
-    done \
-    && for key in \
-      "${TAR_KEYS[@]}"
-    ; do \
-      gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$key" || \
-      gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" ; \
-    done \
-    && for key in \
-      "${CMAKE_KEYS[@]}"
-    ; do \
-      gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" || \
-      gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" ; \
-    done \
-	&& gpg --batch --verify python.tar.xz.asc python.tar.xz \
-	&& rm -rf python.tar.xz.asc \
-	&& mkdir -p /usr/src/python \
-	&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \
-	&& rm python.tar.xz \
-	\
-	&& mkdir -p /usr/src/git \
-	&& tar -xJC /usr/src/git --strip-components=1 -f git.tar.xz \
-	&& rm git.tar.xz \
-	&& gpg --batch --verify tar.tar.xz.sign tar.tar.xz \
-	&& rm -rf tar.tar.xz.sign \
-	&& mkdir -p /usr/src/tar \
-	&& tar -xJC /usr/src/tar --strip-components=1 -f tar.tar.xz \
-	&& rm tar.tar.xz \
-	\
-	&& cd /usr/src/python \
 %%IF VARIANT=7%%
-	&& sed -i 's/PKG_CONFIG openssl /PKG_CONFIG openssl11 /g' configure \
+	&& /_bootstrap/openssl.sh \
 %%ENDIF%%
-	&& gnuArch="$(gcc -dumpmachine)" \
-	&& ./configure \
-		--build="$gnuArch" \
-		--enable-loadable-sqlite-extensions \
-		--enable-shared \
-		--with-system-expat \
-		--with-system-ffi \
-		--without-ensurepip \
-	&& make -j "$(nproc)" \
-	&& make install \
-	&& ldconfig \
-	\
-	&& find /usr/local -depth \
-		\( \
-			\( -type d -a \( -name test -o -name tests \) \) \
-			-o \
-			\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
-		\) -exec rm -rf '{}' + \
-	&& rm -rf /usr/src/python \
-	\
-	&& cd /usr/src/git \
-	&& make prefix=/usr/local V=1 -j $(nproc) all \
-	&& make prefix=/usr/local install \
-	&& rm -rf /usr/src/git \
-	&& cd / \
-	&& mkdir -p /usr/src/pkgconf \
-	&& tar -xJC /usr/src/pkgconf --strip-components=1 -f pkgconf.tar.xz \
-	&& rm pkgconf.tar.xz \
-	&& cd /usr/src/pkgconf \
-	&& ./configure \
-	&& make -j $(nproc) \
-	&& make install \
-	&& rm -rf /usr/src/pkgconf \
-	&& ln -sf /usr/local/bin/pkgconf /usr/local/bin/pkg-config \
-	&& cd /usr/src/tar \
-	&& env FORCE_UNSAFE_CONFIGURE=1 ./configure \
-		--bindir=/usr/local/bin/ \
-		--libexecdir=/usr/local/sbin/ \
-	&& make -j $(nproc) \
-	&& make install \
-	&& rm -rf /usr/src/tar \
-	&& cd / \
-	&& curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- \
-		-y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION \
-	&& chmod -R a+w $RUSTUP_HOME $CARGO_HOME \
-	\
-	&& dpkgArch="$(rpm --eval '%{_arch}')" \
-	&& dpkgArch="$(rpm --eval '%{_arch}')" \
-	&& case "${dpkgArch##*-}" in \
-		x86_64) \
-			NODE_ARCH='x64' \
-			GO_ARCH='amd64' \
-			CMAKE_ARCH='x86_64' \
-			NINJA_ARCH='' \
-			;; \
-		ppc64el) \
-			NODE_ARCH='ppc64le' \
-			GO_ARCH='ppc64le' \
-			CMAKE_ARCH='' \
-			NINJA_ARCH='' \
-			;; \
-		s390x) \
-			NODE_ARCH='s390x' \
-			GO_ARCH='s390x' \
-			CMAKE_ARCH='' \
-			NINJA_ARCH='' \
-			;; \
-		arm64) \
-			NODE_ARCH='arm64' \
-			GO_ARCH='arm64' \
-			CMAKE_ARCH='' \
-			NINJA_ARCH='aarch64' \
-			;; \
-		aarch64) \
-			NODE_ARCH='arm64' \
-			GO_ARCH='arm64' \
-			CMAKE_ARCH='aarch64' \
-			NINJA_ARCH='aarch64' \
-			;; \
-		i386) \
-			NODE_ARCH='x86' \
-			GO_ARCH='386' \
-			CMAKE_ARCH='' \
-			CMAKE_ARCH='' \
-			;; \
-		*) echo "unsupported architecture"; exit 1 ;; \
-	esac \
-	&& curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-${NODE_ARCH}.tar.xz" \
-	&& curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
-	&& gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \
-	&& grep " node-v$NODE_VERSION-linux-${NODE_ARCH}.tar.xz\$" SHASUMS256.txt | sha256sum -c - \
-	&& tar -xJf "node-v$NODE_VERSION-linux-${NODE_ARCH}.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \
-	&& rm "node-v$NODE_VERSION-linux-${NODE_ARCH}.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \
-	&& ln -s /usr/local/bin/node /usr/local/bin/nodejs \
-	&& curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \
-	&& curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \
-	&& gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \
-	&& mkdir -p /opt \
-	&& tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ \
-	&& ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn \
-	&& ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg \
-	&& rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \
-	&& curl --proto '=https' --tlsv1.2 -sSfL \
-		https://go.dev/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz -o /tmp/go.tgz \
-	&& tar -C /usr/local -xzf /tmp/go.tgz \
-	&& rm /tmp/go.tgz \
-	&& curl -fsSL "https://github.com/ninja-build/ninja/archive/refs/tags/v${NINJA_VERSION}.tar.gz" -o /tmp/ninja.tar.gz \
-	&& mkdir -p /usr/src/ninja \
-	&& tar -xzC /usr/src/ninja --strip-components=1 -f /tmp/ninja.tar.gz \
-	&& rm /tmp/ninja.tar.gz \
-	&& cd /usr/src/ninja \
-	&& ./configure.py --bootstrap --verbose \
-	&& cp -a ./ninja /usr/local/bin/ninja \
-	&& curl -fsSLO --compressed "https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION${CMAKE_ARCH:+-linux-$CMAKE_ARCH}.tar.gz" \
-	&& curl -fsSLO --compressed "https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-SHA-256.txt" \
-	&& curl -fsSLO --compressed "https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-SHA-256.txt.asc" \
-	&& gpg --batch --verify cmake-$CMAKE_VERSION-SHA-256.txt.asc cmake-$CMAKE_VERSION-SHA-256.txt \
-	&& grep " cmake-$CMAKE_VERSION${CMAKE_ARCH:+-linux-$CMAKE_ARCH}.tar.gz\$" cmake-$CMAKE_VERSION-SHA-256.txt | sha256sum -c - \
-	&& if [ -n "$CMAKE_ARCH" ]; then \
-		echo $CMAKE_ARCH; \
-		tar -xzf "cmake-$CMAKE_VERSION-linux-$CMAKE_ARCH.tar.gz" -C /usr/local --strip-components=1 --no-same-owner; \
-	else \
-		mkdir cmake-build; \
-		tar -xzf "cmake-$CMAKE_VERSION.tar.gz" -C cmake-build --strip-components=1; \
-		cd cmake-build; \
-		./bootstrap --parallel="$(nproc)"; \
-		make -j "$(nproc)"; \
-		make install; \
-		cd ..; \
-		rm -rf cmake-build; \
-	fi \
-	&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \
-	&& rm -rf "$GNUPGHOME" \
-	\
-	&& python3 --version \
-	&& rustc --version \
-	&& go version \
-	&& node --version \
-	&& npm --version \
-	&& yarn --version \
-	&& git --version \
-	&& tar --version \
-	&& ninja --version
-
-# make some useful symlinks that are expected to exist
-RUN cd /usr/local/bin \
-	&& ln -s idle3 idle \
-	&& ln -s pydoc3 pydoc \
-	&& ln -s python3 python \
-	&& ln -s python3-config python-config
-
-# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION %%PLACEHOLDER%%
-
-RUN set -ex; \
-	\
-	curl -fsSLo get-pip.py 'https://bootstrap.pypa.io/get-pip.py'; \
-	\
-	python get-pip.py \
-		--disable-pip-version-check \
-		"pip==$PYTHON_PIP_VERSION" \
-	; \
-	pip --version; \
-	\
-	find /usr/local -depth \
-		\( \
-			\( -type d -a \( -name test -o -name tests \) \) \
-			-o \
-			\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
-		\) -exec rm -rf '{}' +; \
-	rm -f get-pip.py
-
-%%WRITE_ENTRYPOINT%%
-RUN chmod +x /entrypoint.sh
-
-ENTRYPOINT ["/entrypoint.sh"]
+	&& /_bootstrap/python.sh \
+	&& /_bootstrap/git.sh \
+	&& /_bootstrap/patchelf.sh \
+	&& /_bootstrap/tar.sh \
+	&& /_bootstrap/pkgconf.sh \
+	&& /_bootstrap/rust.sh \
+	&& /_bootstrap/node.sh \
+	&& /_bootstrap/yarn.sh \
+	&& /_bootstrap/go.sh \
+	&& /_bootstrap/meson.sh \
+	&& /_bootstrap/ninja.sh \
+	&& /_bootstrap/cmake.sh