feat(docker): make containers multi-arch

.dagger-ci/daggerci/lib/orchestrator.py

@@ -8,6 +8,7 @@
 import datetime
 import logging
 import os
+import platform
 import re
 import sys
 from typing import Any
@@ -41,6 +42,28 @@ class ContainerTestFailed(Exception):
     """
 
 
+def get_current_platform() -> str:
+    """
+    Get platform string of current machine
+    Examples: 'linux/amd64' or 'windows/arm64'
+    """
+    # Figure out Operating System, aka Platform
+    # Docs: https://docs.python.org/3/library/sys.html#sys.platform
+    current_platform = sys.platform.lower()
+    if current_platform in ("win32", "cygwin"):
+        current_platform = "windows"
+
+    # Figure out CPU
+    # Docs: https://docs.python.org/3/library/platform.html#platform.machine
+    current_arch = platform.machine().lower()
+    if current_arch == "x86_64":
+        current_arch = "amd64"
+    if current_arch == "aarch64":
+        current_arch = "arm64"
+
+    return f"{current_platform}/{current_arch}"
+
+
 class Orchestrator:
     """
     The main class to abstract all actions
@@ -133,6 +156,7 @@ def __init__(
         self.tags = [
             self.tag_sha_short,
             re.sub(r"[\/-]", r"_", self.tag_branch),
+            get_current_platform(),
             # self.tag_timestamp,
         ]
         if self.tag_tag is not None:
@@ -233,25 +257,15 @@ async def __build_test_publish__(
         # =======
         # BUILD
         logging.info("%s/%s: BUILDING", top_element, dockerfile)
-        try:
-            built_docker = await self.__build__(
-                client=client,
-                dockerfile_dir=dockerfile_dir,
-                dockerfile_args=dockerfile_args,
-            )
-        except dagger.ExecError as exc:
-            logging.error("Dagger execution error")
-            self.results.add(top_element, dockerfile, "build", False, exc.message)
-            return
-        except dagger.QueryError as exc:
-            logging.error(
-                "Dagger query error, try this: https://archive.docs.dagger.io/0.9/235290/troubleshooting/#dagger-pipeline-is-unable-to-resolve-host-names-after-network-configuration-changes"
-            )
-            self.results.add(
-                top_element, dockerfile, "build", False, exc.debug_query()
-            )  # type: ignore [no-untyped-call]
+        variants = await self.__build__(
+            client=client,
+            dockerfile=dockerfile,
+            dockerfile_dir=dockerfile_dir,
+            dockerfile_args=dockerfile_args,
+            top_element=top_element,
+        )
+        if not variants:
             return
-        self.results.add(top_element, dockerfile, "build")
 
         # add container specific labels into self.labels
         self.labels["org.opencontainers.image.description"] = (
@@ -262,22 +276,25 @@ async def __build_test_publish__(
         )
 
         # add labels to the container
-        for name, val in self.labels.items():
-            built_docker = await built_docker.with_label(name=name, value=val)
+        for built_docker in variants:
+            for name, val in self.labels.items():
+                built_docker = await built_docker.with_label(name=name, value=val)
 
         logging.info("Docker container labels:")
-        for label in await built_docker.labels():
-            logging.info("label: %s = %s", await label.name(), await label.value())
+        for built_docker in variants:
+            for label in await built_docker.labels():
+                logging.info("label: %s = %s", await label.name(), await label.value())
 
         # =======
         # TEST
         logging.info("%s/%s: TESTING", top_element, dockerfile)
         try:
-            await self.__test__(
-                client=client,
-                test_container=built_docker,
-                test_container_name=dockerfile,
-            )
+            for built_docker in variants:
+                await self.__test__(
+                    client=client,
+                    test_container=built_docker,
+                    test_container_name=dockerfile,
+                )
         except ContainerTestFailed:
             self.results.add(top_element, dockerfile, "test", False)
             return
@@ -294,7 +311,7 @@ async def __build_test_publish__(
             # pylint: enable=attribute-defined-outside-init
             try:
                 await self.__publish__(
-                    container=built_docker,
+                    variants=variants,
                     dockerfile=dockerfile,
                     top_element=top_element,
                 )
@@ -307,18 +324,63 @@ async def __build_test_publish__(
             self.results.add(top_element, dockerfile, "publish", False, "skip")
 
     async def __build__(
-        self, client: dagger.Client, dockerfile_dir: str, dockerfile_args: list[Any]
-    ) -> dagger.Container:
+        self,
+        client: dagger.Client,
+        dockerfile: str,
+        dockerfile_dir: str,
+        dockerfile_args: list[Any],
+        top_element: str,
+    ) -> list[dagger.Container]:
         # dockerfile_args: list[dagger.api.gen.BuildArg]) -> dagger.Container:
         # For some reason I get
         #   "AttributeError: module 'dagger' has no attribute 'api'"
+
+        # pylint: disable=too-many-arguments
         """
         Does the actual building of docker container
         """
+
+        # Initially I wanted to use this setup to build all wanted platforms, but unfortunately
+        #   there are issues with native emulation when building tool-chains for coreboot and edk2.
+        # For that reason we cannot build the multi-arch container as shown in cookbook
+        #   https://docs.dagger.io/cookbook/#build-multi-arch-image
+        # :(
+
+        platforms = [
+            get_current_platform(),
+        ]
         context_dir = client.host().directory(dockerfile_dir)
-        return await context_dir.docker_build(  # type: ignore [no-any-return]
-            build_args=dockerfile_args
-        )
+        platform_variants = []
+
+        for p in platforms:
+            try:
+                logging.info("** building platform: %s", p)
+                container = await context_dir.docker_build(  # type: ignore [no-any-return]
+                    platform=dagger.Platform(p),
+                    build_args=dockerfile_args,
+                )
+                platform_variants.append(container)
+            except dagger.ExecError as exc:
+                logging.error("Dagger execution error")
+                self.results.add(
+                    top_element, dockerfile, f"build {p}", False, exc.message
+                )
+                return []
+            except dagger.QueryError as exc:
+                logging.error(
+                    "Dagger query error, try this: https://archive.docs.dagger.io/0.9/235290/troubleshooting/#dagger-pipeline-is-unable-to-resolve-host-names-after-network-configuration-changes"
+                )
+                self.results.add(
+                    top_element,
+                    dockerfile,
+                    f"build {p}",
+                    False,
+                    exc.debug_query(),
+                )  # type: ignore [no-untyped-call]
+                return []
+            self.results.add(top_element, dockerfile, f"build {p}")
+
+        return platform_variants
 
     async def __test__(
         self,
@@ -384,18 +446,27 @@ async def __test__(
         # No return here, so the execution continues normally
 
     async def __publish__(
-        self, container: dagger.Container, dockerfile: str, top_element: str
+        self,
+        variants: list[dagger.Container],
+        dockerfile: str,
+        top_element: str,
     ) -> None:
         """
         Publish the built container to container registry
         """
+
+        # Get the first container and use it as base
+        container = variants.pop(1)
+
         for tag in self.tags:
             image_ref = await container.with_registry_auth(
                 address=str(self.container_registry),
                 username=str(self.container_registry_username),
                 secret=self.secret_token,
             ).publish(
-                f"{self.container_registry}/{self.organization}/{self.project_name}/{dockerfile}:{tag}"
+                f"{self.container_registry}/{self.organization}/{self.project_name}/{dockerfile}:{tag}",
+                # add remaining containers:
+                platform_variants=variants,
             )
             logging.info(
                 "%s/%s: Published image to: %s", top_element, dockerfile, image_ref

.dagger-ci/daggerci/tests/test_orchestrator.py

@@ -41,8 +41,14 @@ async def test__orchestrator__broken_dockerfile(
     result = await my_orchestrator.build_test_publish()
     assert "services" in result.results
     assert "coreboot_4.19" in result.results["services"]
-    assert "build" in result.results["services"]["coreboot_4.19"]
-    assert result.results["services"]["coreboot_4.19"]["build"] is False
+
+    # because of multi-platform nature we have to be flexible
+    build_found = False
+    for key, _ in result.results["services"]["coreboot_4.19"].items():
+        if re.match("build .*", key) and not re.match(".*_msg$", key):
+            build_found = True
+            assert result.results["services"]["coreboot_4.19"][key] is False
+    assert build_found
 
 
 @pytest.mark.slow
@@ -148,39 +154,39 @@ async def test__orchestrator__multi_comprehensive_build(
             None,
             "branch1",
             None,
-            ["sha12ab", "branch1"],
+            ["sha12ab", "branch1", "linux/amd64"],
         ),
         (
             "sha12ab",
             "sha12ab0123456789abcdef0123456789abcdef0",
             "v1.0",
             "master",
             None,
-            ["sha12ab", "v1.0", "master", "latest"],
+            ["sha12ab", "v1.0", "master", "latest", "linux/amd64"],
         ),
         (
             "sha12ab",
             "sha12ab0123456789abcdef0123456789abcdef0",
             "v1.1.0-3-11dd3b9",
             "branch1",
             None,
-            ["sha12ab", "branch1"],
+            ["sha12ab", "branch1", "linux/amd64"],
         ),
         (
             "sha12ab",
             "sha12ab0123456789abcdef0123456789abcdef0",
             "v1.1.0-3-11dd3b9",
             "branch1",
             "34",
-            ["sha12ab", "branch1", "pull_request_34"],
+            ["sha12ab", "branch1", "pull_request_34", "linux/amd64"],
         ),
         (
             "sha12ab",
             "sha12ab0123456789abcdef0123456789abcdef0",
             "v1.1.0-3-11dd3b9",
             "feature/new-stuff",
             "34",
-            ["sha12ab", "feature_new_stuff", "pull_request_34"],
+            ["sha12ab", "feature_new_stuff", "pull_request_34", "linux/amd64"],
         ),
     ],
 )

.github/workflows/docker-build-and-test.yml

@@ -2,7 +2,7 @@
 # Test built docker images by building simple projects inside them
 
 name: dagger
 on:
   pull_request:
     paths:
       - '.dagger-ci'
@@ -31,11 +31,18 @@
 
 jobs:
   build:
-    name: build_test_publish
-    runs-on: ubuntu-latest
+    name: build_${{ matrix.dockerfile }}_${{ matrix.os }}
+    runs-on:
+      group: ubuntu-runners
+      labels: [latest, 4core, ${{ matrix.arch }} ]
     timeout-minutes: 120
     strategy:
       matrix:
+        arch:
+          [
+            'amd64',
+            'arm64'
+          ]
         dockerfile:
           [
             'coreboot_4.19',
@@ -123,3 +130,37 @@
           GITHUB_REGISTRY: ${{ env.REGISTRY }}
           GITHUB_ACTOR: ${{ github.actor }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  merge-multi-arch:
+    name: merge_multiarch
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        dockerfile:
+          [
+            'coreboot_4.19',
+            'coreboot_4.20.1',
+            'coreboot_4.21',
+            'coreboot_4.22.01',
+            'coreboot_24.02',
+            'coreboot_24.02.01',
+            'coreboot_24.05',
+            'edk2-stable202008',
+            'edk2-stable202105',
+            'edk2-stable202111',
+            'edk2-stable202205',
+            'edk2-stable202208',
+            'edk2-stable202211',
+            'edk2-stable202408',
+            'linux_6.1.45',
+            'linux_6.1.111',
+            'linux_6.6.52',
+            'linux_6.9.9',
+            'linux_6.11',
+            'udk2017',
+            'uroot_0.14.0'
+          ]
+    needs: ['build']
+    steps:
+      - run: |
+          echo "${{ matrix.dockerfile}}"

docker/coreboot/Dockerfile

@@ -19,7 +19,8 @@ ENV TOOLSDIR=/tools
 # Use coreboot mirror
 ENV BUILDGCC_OPTIONS=-m
 
-RUN apt-get update && \
+RUN arch="$(dpkg --print-architecture)" && arch="${arch##*-}"; \
+    apt-get update && \
     apt-get install -y --no-install-recommends \
         acpica-tools \
         bc \
@@ -31,7 +32,6 @@ RUN apt-get update && \
         git \
         gnat \
         imagemagick \
-        iucode-tool \
         libelf-dev \
         libncurses5-dev \
         libnss3-dev \
@@ -48,6 +48,10 @@ RUN apt-get update && \
         uuid-dev \
         zlib1g-dev \
     && \
+    if [ "${arch}" = 'amd64' ]; then \
+        apt-get install -y --no-install-recommends \
+            iucode-tool; \
+    fi; \
     apt-get install -y --no-install-recommends \
         less \
         nano \