Merge pull request #2351 from septatrix/feature/oci-output-format

Add support for oci-dir output (fixes #1865)

mkosi/__init__.py

@@ -21,7 +21,7 @@
 import uuid
 from collections.abc import Iterator, Mapping, Sequence
 from pathlib import Path
-from typing import Optional, TextIO, Union, cast
+from typing import Optional, Union, cast
 
 from mkosi.archive import extract_tar, make_cpio, make_tar
 from mkosi.burn import run_burn
@@ -76,6 +76,7 @@
     flock,
     flock_or_die,
     format_rlimit,
+    hash_file,
     make_executable,
     one_zero,
     read_env_file,
@@ -2457,17 +2458,6 @@ def copy_initrd(context: Context) -> None:
         break
 
 
-def hash_file(of: TextIO, path: Path) -> None:
-    bs = 16 * 1024**2
-    h = hashlib.sha256()
-
-    with path.open("rb") as sf:
-        while (buf := sf.read(bs)):
-            h.update(buf)
-
-    of.write(h.hexdigest() + " *" + path.name + "\n")
-
-
 def calculate_sha256sum(context: Context) -> None:
     if not context.config.checksum:
         return
@@ -2478,7 +2468,7 @@ def calculate_sha256sum(context: Context) -> None:
     with complete_step("Calculating SHA256SUMS…"):
         with open(context.workspace / context.config.output_checksum, "w") as f:
             for p in context.staging.iterdir():
-                hash_file(f, p)
+                print(hash_file(p) + " *" + p.name, file=f)
 
         (context.workspace / context.config.output_checksum).rename(context.staging / context.config.output_checksum)
 
@@ -3276,6 +3266,103 @@ def make_disk(
     return make_image(context, msg=msg, skip=skip, split=split, tabs=tabs, root=context.root, definitions=definitions)
 
 
+def make_oci(context: Context, root_layer: Path, dst: Path) -> None:
+    ca_store = dst / "blobs" / "sha256"
+    with umask(~0o755):
+        ca_store.mkdir(parents=True)
+
+    layer_diff_digest = hash_file(root_layer)
+    maybe_compress(
+        context,
+        context.config.compress_output,
+        context.staging / "rootfs.layer",
+        # Pass explicit destination to suppress adding an extension
+        context.staging / "rootfs.layer",
+    )
+    layer_digest = hash_file(root_layer)
+    root_layer.rename(ca_store / layer_digest)
+
+    creation_time = (
+        datetime.datetime.fromtimestamp(context.config.source_date_epoch, tz=datetime.timezone.utc)
+        if context.config.source_date_epoch is not None
+        else datetime.datetime.now(tz=datetime.timezone.utc)
+    ).isoformat()
+
+    oci_config = {
+        "created": creation_time,
+        "architecture": context.config.architecture.to_oci(),
+        # Name of the operating system which the image is built to run on as defined by
+        # https://github.com/opencontainers/image-spec/blob/v1.0.2/config.md#properties.
+        "os": "linux",
+        "rootfs": {
+            "type": "layers",
+            "diff_ids": [f"sha256:{layer_diff_digest}"],
+        },
+        "config": {
+            "Cmd": [
+                "/sbin/init",
+                *context.config.kernel_command_line,
+            ],
+        },
+        "history": [
+            {
+                "created": creation_time,
+                "comment": "Created by mkosi",
+            },
+        ],
+    }
+    oci_config_blob = json.dumps(oci_config)
+    oci_config_digest = hashlib.sha256(oci_config_blob.encode()).hexdigest()
+    with umask(~0o644):
+        (ca_store / oci_config_digest).write_text(oci_config_blob)
+
+    layer_suffix = context.config.compress_output.oci_media_type_suffix()
+    oci_manifest = {
+        "schemaVersion": 2,
+        "mediaType": "application/vnd.oci.image.manifest.v1+json",
+        "config": {
+            "mediaType": "application/vnd.oci.image.config.v1+json",
+            "digest": f"sha256:{oci_config_digest}",
+            "size": (ca_store / oci_config_digest).stat().st_size,
+        },
+        "layers": [
+            {
+                "mediaType": f"application/vnd.oci.image.layer.v1.tar{layer_suffix}",
+                "digest": f"sha256:{layer_digest}",
+                "size": (ca_store / layer_digest).stat().st_size,
+            }
+        ],
+        "annotations": {
+            "io.systemd.mkosi.version": __version__,
+            **({
+                "org.opencontainers.image.version": context.config.image_version,
+            } if context.config.image_version else {}),
+        }
+    }
+    oci_manifest_blob = json.dumps(oci_manifest)
+    oci_manifest_digest = hashlib.sha256(oci_manifest_blob.encode()).hexdigest()
+    with umask(~0o644):
+        (ca_store / oci_manifest_digest).write_text(oci_manifest_blob)
+
+        (dst / "index.json").write_text(
+            json.dumps(
+                {
+                    "schemaVersion": 2,
+                    "mediaType": "application/vnd.oci.image.index.v1+json",
+                    "manifests": [
+                        {
+                            "mediaType": "application/vnd.oci.image.manifest.v1+json",
+                            "digest": f"sha256:{oci_manifest_digest}",
+                            "size": (ca_store / oci_manifest_digest).stat().st_size,
+                        }
+                    ],
+                }
+            )
+        )
+
+        (dst / "oci-layout").write_text(json.dumps({"imageLayoutVersion": "1.0.0"}))
+
+
 def make_esp(context: Context, uki: Path) -> list[Partition]:
     if not (arch := context.config.architecture.to_efi()):
         die(f"Architecture {context.config.architecture} does not support UEFI")
@@ -3577,6 +3664,17 @@ def build_image(context: Context) -> None:
             tools=context.config.tools(),
             sandbox=context.sandbox,
         )
+    elif context.config.output_format == OutputFormat.oci:
+        make_tar(
+            context.root, context.staging / "rootfs.layer",
+            tools=context.config.tools(),
+            sandbox=context.sandbox,
+        )
+        make_oci(
+            context,
+            context.staging / "rootfs.layer",
+            context.staging / context.config.output_with_format,
+        )
     elif context.config.output_format == OutputFormat.cpio:
         make_cpio(
             context.root, context.staging / context.config.output_with_format,

mkosi/config.py

@@ -173,6 +173,7 @@ class OutputFormat(StrEnum):
     sysext    = enum.auto()
     tar       = enum.auto()
     uki       = enum.auto()
+    oci       = enum.auto()
 
     def extension(self) -> str:
         return {
@@ -205,6 +206,7 @@ class Compression(StrEnum):
     xz   = enum.auto()
     bz2  = enum.auto()
     gz   = enum.auto()
+    gzip = "gz"
     lz4  = enum.auto()
     lzma = enum.auto()
 
@@ -216,6 +218,18 @@ def extension(self) -> str:
             Compression.zstd: ".zst"
         }.get(self, f".{self}")
 
+    def oci_media_type_suffix(self) -> str:
+        suffix = {
+            Compression.none: "",
+            Compression.gz:   "+gzip",
+            Compression.zstd: "+zstd",
+        }.get(self)
+
+        if not suffix:
+            die(f"Compression {self} not supported for OCI layers")
+
+        return suffix
+
 
 class DocFormat(StrEnum):
     auto     = enum.auto()
@@ -379,6 +393,28 @@ def to_qemu(self) -> str:
 
         return a
 
+    def to_oci(self) -> str:
+        a = {
+            Architecture.arm         : "arm",
+            Architecture.arm64       : "arm64",
+            Architecture.loongarch64 : "loong64",
+            Architecture.mips64_le   : "mips64le",
+            Architecture.mips_le     : "mipsle",
+            Architecture.ppc         : "ppc",
+            Architecture.ppc64       : "ppc64",
+            Architecture.ppc64_le    : "ppc64le",
+            Architecture.riscv32     : "riscv",
+            Architecture.riscv64     : "riscv64",
+            Architecture.s390x       : "s390x",
+            Architecture.x86         : "386",
+            Architecture.x86_64      : "amd64",
+        }.get(self)
+
+        if not a:
+            die(f"Architecture {self} not supported by OCI")
+
+        return a
+
     def default_serial_tty(self) -> str:
         return {
             Architecture.arm      : "ttyAMA0",
@@ -634,6 +670,8 @@ def config_default_compression(namespace: argparse.Namespace) -> Compression:
             return Compression.xz
         else:
             return Compression.zstd
+    elif namespace.output_format == OutputFormat.oci:
+        return Compression.gz
     else:
         return Compression.none
 

mkosi/resources/mkosi.md

@@ -672,7 +672,8 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`,
   archive is generated), `disk` (a block device OS image with a GPT
   partition table), `uki` (a unified kernel image with the OS image in
   the `.initrd` PE section), `esp` (`uki` but wrapped in a disk image
-  with only an ESP partition), `sysext`, `confext`, `portable` or `none`
+  with only an ESP partition), `oci` (a directory compatible with the
+  OCI image specification), `sysext`, `confext`, `portable` or `none`
   (the OS image is solely intended as a build image to produce another
   artifact).
 
@@ -710,11 +711,12 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`,
 : Configure compression for the resulting image or archive. The argument can be
   either a boolean or a compression algorithm (`xz`, `zstd`). `zstd`
   compression is used by default, except CentOS and derivatives up to version
-  8, which default to `xz`. Note that when applied to block device image types,
+  8, which default to `xz`, and OCI images, which default to `gzip`.
+  Note that when applied to block device image types,
   compression means the image cannot be started directly but needs to be
   decompressed first. This also means that the `shell`, `boot`, `qemu` verbs
   are not available when this option is used. Implied for `tar`, `cpio`, `uki`,
-  and `esp`.
+  `esp`, and `oci`.
 
 `CompressLevel=`, `--compress-level=`
 

mkosi/util.py

@@ -7,6 +7,7 @@
 import errno
 import fcntl
 import functools
+import hashlib
 import importlib
 import importlib.resources
 import itertools
@@ -299,3 +300,15 @@ def _write_contents(target, source):
             p.parent.chmod(0o755)
 
         yield p
+
+
+def hash_file(path: Path) -> str:
+    # TODO Replace with hashlib.file_digest after dropping support for Python 3.10.
+    bs = 16 * 1024**2
+    h = hashlib.sha256()
+
+    with path.open("rb") as sf:
+        while (buf := sf.read(bs)):
+            h.update(buf)
+
+    return h.hexdigest()

tests/test_boot.py

@@ -57,7 +57,7 @@ def test_format(config: Image.Config, format: OutputFormat) -> None:
         if image.config.distribution == Distribution.rhel_ubi:
             return
 
-        if format in (OutputFormat.tar, OutputFormat.none) or format.is_extension_image():
+        if format in (OutputFormat.tar, OutputFormat.oci, OutputFormat.none) or format.is_extension_image():
             return
 
         if format == OutputFormat.directory and not find_virtiofsd():