[v15] build: Update buildbox-ng for FIPS builds

* buildbox: Cache git fetch directories

When fetching sources via git, first clone into the download directory
before cloning a second time from the download directory to the final
source build location. This second clone is pretty quick so does not add
much extra time in the base case, but subsequent clones are much faster
without needing the network.

This is particularly useful during development of the buildbox as the
download directory is a persistent cache, so everything only needs to be
cloned once, once and for all, rather than four times each build (once
per architecture, and on each and every build).

* buildbox: Add clang 12.0.0 to buildbox

Add clang 12.0.0 to the buildbox in `/opt/clang` so that the `boring-rs`
crate can build `boringssl` for the FIPS build of Teleport. This
specific version of clang is required for FIPS compliance.

To make this work we need to ensure some arguments are always passed to
clang (`--gcc-toolchain` and `--sysroot`) so we also add a front-end
shell script that allows extra arguments to be passed to clang via the
environment.

Also add a few symlinks as it seems cmake that builds boringssl needs
some clang/llvm tools to be specifically named and located:

    /opt/clang/bin/clang++-12
    /usr/bin/llvm-ar-12
    /usr/bin/llvm-ranlib-12

* build: Update Makefiles to enable FIPS build with buildbox-ng

Update the Makefiles so that we can use buildbox-ng to make a FIPS
release build of Teleport.

This is done by adding a new release target `release-ng-amd64-fips` that
builds just the enterprise edition of Teleport with FIPS enabled and
updating the top-level `Makefile` to set some environment variables
needed for a FIPS build and a target to build just an enterprise
release.

Makefile

@@ -80,6 +80,18 @@ ifneq ("$(FIPS)","")
 FIPS_TAG := fips
 FIPS_MESSAGE := with-FIPS-support
 RELEASE = teleport-$(GITTAG)-$(OS)-$(ARCH)-fips-bin
+GOEXPERIMENT = boringcrypto
+OPENSSL_FIPS = 1
+export GOEXPERIMENT OPENSSL_FIPS
+ifeq ($(BUILDBOX_MODE),cross)
+# We need to set CGO_ENABLED=0 when building rdpclient as the build of
+# boring-sys builds and runs a Go program as part of its integrity testing.
+# (https://github.com/google/boringssl/blob/master/crypto/fipsmodule/FIPS.md#integrity-testing)
+# If CGO_ENABLED=1, this fails for odd reasons (it tries to use the cross
+# assembler to build ASM for the host).
+# It also needs to know the cross-compiler sysroot to properly cross-compile.
+RDPCLIENT_ENV = CGO_ENABLED=0 BORING_BSSL_FIPS_SYSROOT=$(CROSSTOOLNG_SYSROOT)
+endif
 endif
 
 # Look for the PAM header "security/pam_appl.h" to determine if we should
@@ -402,7 +414,8 @@ endif
 .PHONY: rdpclient
 rdpclient:
 ifeq ("$(with_rdpclient)", "yes")
-	cargo build -p rdp-client $(if $(FIPS),--features=fips) --release --locked $(CARGO_TARGET)
+	$(RDPCLIENT_ENV) \
+		cargo build -p rdp-client $(if $(FIPS),--features=fips) --release --locked $(CARGO_TARGET)
 endif
 
 # Build libfido2 and dependencies for MacOS. Uses exported C_ARCH variable defined earlier.
@@ -465,16 +478,14 @@ clean-ui:
 	rm -rf web/packages/teleterm/build
 	find . -type d -name node_modules -prune -exec rm -rf {} \;
 
-#
-# make release - Produces a binary release tarball.
-#
-
 # RELEASE_DIR is where release artifact files are put, such as tarballs, packages, etc.
 $(RELEASE_DIR):
 	mkdir -p $@
 
-.PHONY:
-export
+#
+# make release - Produces a binary release tarball.
+#
+.PHONY: release
 release:
 	@echo "---> OSS $(RELEASE_MESSAGE)"
 ifeq ("$(OS)", "windows")
@@ -485,6 +496,12 @@ else
 	$(MAKE) --no-print-directory release-unix
 endif
 
+.PHONY: release-ent
+release-ent:
+ifneq (,$(wildcard e/Makefile))
+	$(MAKE) -C e release
+endif
+
 # These are aliases used to make build commands uniform.
 .PHONY: release-amd64
 release-amd64:

build.assets/Makefile

@@ -603,9 +603,11 @@ release: $(BUILDBOX_TARGET)
 		/usr/bin/make $(RELEASE_TARGET) -e ADDFLAGS="$(ADDFLAGS)" OS=$(OS) ARCH=$(ARCH) RUNTIME=$(GOLANG_VERSION) FIDO2=$(FIDO2) PIV=$(PIV) REPRODUCIBLE=yes
 
 
-.PHONY: release-ng-amd64 release-ng-arm64 release-ng-386 release-ng-arm
+.PHONY: release-ng-amd64 release-ng-amd64-fips release-ng-arm64 release-ng-386 release-ng-arm
 release-ng-amd64:
 	$(MAKE) release-ng ARCH=amd64 FIDO2=yes PIV=yes
+release-ng-amd64-fips:
+	$(MAKE) release-ng ARCH=amd64 FIDO2=yes PIV=yes FIPS=yes RELEASE_TARGET=release-ent
 release-ng-arm64:
 	$(MAKE) release-ng ARCH=arm64 FIDO2=yes PIV=yes
 release-ng-386:
@@ -614,19 +616,21 @@ release-ng-arm:
 	$(MAKE) release-ng ARCH=arm
 
 .PHONY: release-ng
+release-ng: RELEASE_TARGET = release-unix-preserving-webassets
 release-ng: webassets buildbox-ng
 	docker run --rm --interactive $(shell test -t 0 && echo --tty) \
 		--volume $(shell pwd)/..:/home/teleport \
 		--workdir /home/teleport \
 		--user $(shell id -u):$(shell id -g) \
 		$(BUILDBOX_NG) \
-		make -e release-unix-preserving-webassets \
+		make -e $(RELEASE_TARGET) \
 			ADDFLAGS="$(ADDFLAGS)" \
 			OS="$(OS)" \
 			ARCH="$(ARCH)" \
 			RUNTIME="$(GOLANG_VERSION)" \
 			FIDO2="$(FIDO2)" \
 			PIV="$(PIV)" \
+			FIPS="$(FIPS)" \
 			REPRODUCIBLE=yes
 
 #

build.assets/buildbox/Dockerfile

@@ -100,6 +100,31 @@ RUN curl --proto =https --tlsv1.2 -fsSL https://sh.rustup.rs | \
 		arm-unknown-linux-gnueabihf \
 		wasm32-unknown-unknown
 
+# ----------------------------------------------------------------------------
+# Clang 12.0.0 for FIPS builds of boring-rs
+
+FROM base AS clang
+
+# libtinfo5 required to run clang to test it works.
+# xz-utils for decompressing the clang tarball.
+RUN apt-get update \
+    && apt-get install -y libtinfo5 xz-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN install -d -m 0775 -o buildbox -g buildbox /opt/clang
+USER buildbox
+
+# TODO(camscale): Verify signature of download.
+RUN \
+	case "$(uname -m)" in \
+		aarch64|arm64) SUFFIX='aarch64-linux-gnu.tar.xz' ;; \
+		x86_64|amd64) SUFFIX='x86_64-linux-gnu-ubuntu-20.04.tar.xz' ;; \
+		*) echo "Unsupported architecture for clang: $(uname -m)" >&2; exit 1 ;; \
+	esac; \
+	curl -fsSL "https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.0/clang+llvm-12.0.0-${SUFFIX}" | \
+	tar -C /opt/clang -xJ --strip-components=1 && \
+	/opt/clang/bin/clang --version
+
 # ----------------------------------------------------------------------------
 # buildbox image
 #
@@ -113,11 +138,11 @@ RUN apt-get update && apt-get install -y \
     automake \
     autopoint \
     bison \
-    clang-12 \
     cmake \
     flex \
     gettext \
     git \
+    libtinfo5 \
     libtool \
     make \
     ninja-build \
@@ -131,13 +156,33 @@ RUN apt-get update && apt-get install -y \
 
 RUN install -d -m 1777 -o teleport -g teleport /tmp/build
 
+# The boring-rs build wants llvm-{ar,ranlib}-12 in /usr/bin. The
+# clang install has /opt/clang/bin/llvm-{ar,ranlib}. Create /usr/bin
+# symlinks while we're still root
+RUN \
+	ln -nsf /opt/clang/bin/llvm-ar /usr/bin/llvm-ar-12 && \
+	ln -nsf /opt/clang/bin/llvm-ranlib /usr/bin/llvm-ranlib-12
+
 USER buildbox
 
 # Copy compilers from other images
 ARG THIRDPARTY_DIR=/opt/thirdparty
 COPY --from=thirdparty ${THIRDPARTY_DIR} ${THIRDPARTY_DIR}
 COPY --from=rust /opt/rust /opt/rust
 COPY --from=go /opt/go /opt/go
+COPY --from=clang /opt/clang /opt/clang
+
+# The boring-rs build uses cmake which wants clang++ to be called clang++-12.
+RUN ln -nsf clang /opt/clang/bin/clang++-12
+
+# We need a clang front-end script to set some command line args to properly
+# find/use the appropriate cross-compiling gcc toolchain, to build boring-rs
+# in FIPS mode.
+COPY clang-12.sh /opt/clang/bin
+RUN \
+	cd /opt/clang/bin && \
+	mv clang-12 clang-12.bin && \
+	ln -s clang-12.sh clang-12
 
 # Set RUSTUP_HOME so cargo does not warn/error about not finding it at ~/.cargo
 ENV RUSTUP_HOME=/opt/rust
@@ -169,7 +214,7 @@ ENV GOCACHE=/tmp/build/go/build
 
 # Add the writable cargo and go bin directories to the path so we will find
 # binaries build with `cargo install` and `go install` during a build.
-ENV PATH=${CARGO_HOME}/bin:${GOPATH}/bin:${PATH}
+ENV PATH=${CARGO_HOME}/bin:${GOPATH}/bin:/opt/clang/bin:${PATH}
 
 # Set a var so the build system can know it's running in this buildbox.
 ENV BUILDBOX_MODE=cross

build.assets/buildbox/Dockerfile-thirdparty

@@ -90,6 +90,7 @@ RUN make -f crosstoolng.mk install-crosstoolng && \
 
 FROM crosstoolng AS compilers
 
+# crosstool-NG
 RUN --mount=type=cache,id=download,uid=${BUILDBOX_UID},target=${THIRDPARTY_DIR}/download \
 	make -f crosstoolng.mk crosstoolng-build ARCH=amd64 && \
 	rm -rf ${THIRDPARTY_DIR}/amd64/crosstoolng

build.assets/buildbox/buildbox-common.mk

@@ -48,23 +48,25 @@ $(THIRDPARTY_SRCDIR) $(THIRDPARTY_HOST_SRCDIR) $(THIRDPARTY_DLDIR):
 tp-git-ref = $($*_GIT_REF)
 tp-git-repo = $($*_GIT_REPO)
 tp-git-ref-hash = $($*_GIT_REF_HASH)
+tp-git-dl-dir = $(THIRDPARTY_DLDIR)/$*-$($*_VERSION)
 tp-git-src-dir = $($*_SRCDIR)
 define tp-git-fetch-cmd
-	git -C "$(dir $(tp-git-src-dir))" \
+	git -C "$(dir $(tp-git-dl-dir))" \
 		-c advice.detachedHead=false clone --depth=1 \
-		--branch=$(tp-git-ref) $(tp-git-repo) "$(tp-git-src-dir)"
+		--branch=$(tp-git-ref) $(tp-git-repo) "$(tp-git-dl-dir)"
 endef
 
 # Fetch source via git.
 fetch-git-%:
-	mkdir -p $(dir $(tp-git-src-dir))
-	$(if $(wildcard $(tp-git-src-dir)),,$(tp-git-fetch-cmd))
-	@if [ "$$(git -C "$(tp-git-src-dir)" rev-parse HEAD)" != "$(tp-git-ref-hash)" ]; then \
+	mkdir -p $(dir $(tp-git-src-dir)) $(tp-git-dl-dir)
+	$(if $(wildcard $(tp-git-dl-dir)),,$(tp-git-fetch-cmd))
+	@if [ "$$(git -C "$(tp-git-dl-dir)" rev-parse HEAD)" != "$(tp-git-ref-hash)" ]; then \
 		echo "Found unexpected HEAD commit for $(1)"; \
 		echo "Expected: $(tp-git-ref-hash)"; \
-		echo "Got: $$(git -C "$(tp-git-src-dir)" rev-parse HEAD)"; \
+		echo "Got: $$(git -C "$(tp-git-dl-dir)" rev-parse HEAD)"; \
 		exit 1; \
 	fi
+	git clone $(tp-git-dl-dir) "$(tp-git-src-dir)"
 
 # vars for fetch-https-%. `$*` represents the `%` match.
 tp-download-url = $($*_DOWNLOAD_URL)

build.assets/buildbox/clang-12.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec clang-12.bin ${CLANG_EXTRA_ARGS-} "$@"

build.assets/buildbox/cross-compile.mk

@@ -21,6 +21,10 @@ CROSSTOOLNG_TARGET_arm64 = aarch64-unknown-linux-gnu
 CROSSTOOLNG_TARGET_386 = i686-unknown-linux-gnu
 CROSSTOOLNG_TARGET_arm = arm-unknown-linux-gnueabihf
 
+# Define some vars that locate the installation of the toolchain.
+CROSSTOOLNG_TOOLCHAIN = $(THIRDPARTY_HOST_PREFIX)/$(CROSSTOOLNG_TARGET)
+CROSSTOOLNG_SYSROOT = $(CROSSTOOLNG_TOOLCHAIN)/$(CROSSTOOLNG_TARGET)/sysroot
+
 # Define environment variables used by gcc, clang and make to find the
 # appropriate compiler and third party libraries.
 export C_INCLUDE_PATH = $(THIRDPARTY_PREFIX)/include
@@ -32,6 +36,14 @@ export LD = $(CROSSTOOLNG_TARGET)-ld
 
 CROSS_VARS = C_INCLUDE_PATH LIBRARY_PATH PKG_CONFIG_PATH CC CXX LD
 
+# Clang needs to find the gcc toolchain libraries that are not in the sysroot.
+# These extra args are used by the clang-12.sh front-end script so clang is
+# always invoked with the correct location for the GCC cross toolchain.
+# This is used for the boring-rs crate to build boringssl in FIPS mode.
+export CLANG_EXTRA_ARGS = --gcc-toolchain=$(CROSSTOOLNG_TOOLCHAIN) --sysroot=$(CROSSTOOLNG_SYSROOT)
+
+CROSS_VARS += CLANG_EXTRA_ARGS
+
 # arm64 has linking issues using the binutils linker when building the
 # Enterprise Teleport binary ("relocation truncated to fit: R_AARCH64_CALL26
 # against symbol") that is resolved by using the gold linker. Ensure that