diff --git a/.github/workflows/build_charm.yaml b/.github/workflows/build_charm.yaml index d3c98357..c358c974 100644 --- a/.github/workflows/build_charm.yaml +++ b/.github/workflows/build_charm.yaml @@ -82,12 +82,12 @@ jobs: build: strategy: matrix: - bases_index: ${{ fromJSON(needs.collect-bases.outputs.bases) }} - name: 'Build charm | base #${{ matrix.bases_index }}' + base: ${{ fromJSON(needs.collect-bases.outputs.bases) }} + name: 'Build charm | base #${{ matrix.base.index }}' needs: - get-workflow-version - collect-bases - runs-on: ubuntu-latest + runs-on: ${{ matrix.base.runner }} timeout-minutes: 120 steps: - name: Install CLI @@ -130,7 +130,7 @@ jobs: - run: snap list - name: Get pack command id: pack-command - run: get-pack-command --cache='${{ inputs.cache }}' --charm-directory='${{ inputs.path-to-charm-directory }}' --bases-index='${{ matrix.bases_index }}' + run: get-pack-command --cache='${{ inputs.cache }}' --charm-directory='${{ inputs.path-to-charm-directory }}' --bases-index='${{ matrix.base.index }}' - name: Pack charm id: pack working-directory: ${{ inputs.path-to-charm-directory }} @@ -139,14 +139,14 @@ jobs: if: ${{ failure() && steps.pack.outcome == 'failure' }} uses: actions/upload-artifact@v4 with: - name: logs-charmcraft-build-${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-base-${{ matrix.bases_index }} + name: logs-charmcraft-build-${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-base-${{ matrix.base.index }} path: ~/.local/state/charmcraft/log/ if-no-files-found: error - run: touch .empty - name: Upload charm package uses: actions/upload-artifact@v4 with: - name: ${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-base-${{ matrix.bases_index }} + name: ${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-base-${{ matrix.base.index }} # .empty file required to preserve directory structure # See https://github.com/actions/upload-artifact/issues/344#issuecomment-1379232156 path: | diff --git a/python/cli/data_platform_workflows_cli/charmcraft.py b/python/cli/data_platform_workflows_cli/charmcraft.py new file mode 100644 index 00000000..e1897b7a --- /dev/null +++ b/python/cli/data_platform_workflows_cli/charmcraft.py @@ -0,0 +1,6 @@ +import enum + + +class Architecture(str, enum.Enum): + X64 = "amd64" + ARM64 = "arm64" diff --git a/python/cli/data_platform_workflows_cli/collect_charm_bases.py b/python/cli/data_platform_workflows_cli/collect_charm_bases.py index 54bd8df8..d2433147 100644 --- a/python/cli/data_platform_workflows_cli/collect_charm_bases.py +++ b/python/cli/data_platform_workflows_cli/collect_charm_bases.py @@ -11,6 +11,19 @@ import yaml +from . import charmcraft + +RUNNERS = { + charmcraft.Architecture.X64: "ubuntu-latest", + charmcraft.Architecture.ARM64: [ + "self-hosted", + "data-platform", + "ubuntu", + "ARM64", + "4cpu16ram", + ], +} + def main(): logging.basicConfig(level=logging.INFO, stream=sys.stdout) @@ -21,7 +34,17 @@ def main(): yaml_data = yaml.safe_load( pathlib.Path(args.charm_directory, "charmcraft.yaml").read_text() ) - bases = [index for index, _ in enumerate(yaml_data["bases"])] + # GitHub runner for each base + runners = [] + for base in yaml_data["bases"]: + # Bases format: https://discourse.charmhub.io/t/charmcraft-bases-provider-support/4713 + architectures = (base.get("build-on") or base).get("architectures", ["amd64"]) + assert ( + len(architectures) == 1 + ), f"Multiple architectures ({architectures}) in one (charmcraft.yaml) base not supported. Use one base per architecture" + architecture = charmcraft.Architecture(architectures[0]) + runners.append(RUNNERS[architecture]) + bases = [{"index": index, "runner": runner} for index, runner in enumerate(runners)] logging.info(f"Collected {bases=}") default_prefix = ( f'packed-charm-cache-{args.cache}-{args.charm_directory.replace("/", "-")}' diff --git a/python/cli/data_platform_workflows_cli/release_charm.py b/python/cli/data_platform_workflows_cli/release_charm.py index 4e84eaaf..aa492c91 100644 --- a/python/cli/data_platform_workflows_cli/release_charm.py +++ b/python/cli/data_platform_workflows_cli/release_charm.py @@ -9,6 +9,8 @@ import yaml +from . import charmcraft + @dataclasses.dataclass class OCIResource: @@ -47,75 +49,107 @@ def main(): charm_directory = pathlib.Path(args.charm_directory) # Upload charm file(s) & store revision - charm_revisions: list[int] = [] + charm_revisions: dict[charmcraft.Architecture, list[int]] = {} for charm_file in charm_directory.glob("*.charm"): - logging.info(f"Uploading {charm_file=}") + # Examples of `charm_file.name`: + # - "mysql-router-k8s_ubuntu-22.04-amd64.charm" + # - "mysql-router-k8s_ubuntu-22.04-amd64-arm64.charm" + architectures = ( + charm_file.name.split("_")[-1].removesuffix(".charm").split("-")[2:] + ) + assert ( + len(architectures) == 1 + ), f"Multiple architectures ({architectures}) in one (charmcraft.yaml) base not supported. Use one base per architecture" + architecture = charmcraft.Architecture(architectures[0]) + logging.info(f"Uploading {charm_file=} {architecture=}") output = run(["charmcraft", "upload", "--format", "json", charm_file]) revision: int = json.loads(output)["revision"] logging.info(f"Uploaded charm {revision=}") - charm_revisions.append(revision) + charm_revisions.setdefault(architecture, []).append(revision) assert len(charm_revisions) > 0, "No charm packages found" metadata_file = yaml.safe_load((charm_directory / "metadata.yaml").read_text()) charm_name = metadata_file["name"] # (Only for Kubernetes charms) upload OCI image(s) & store revision - oci_resources: list[OCIResource] = [] + oci_resources: dict[charmcraft.Architecture, list[OCIResource]] = {} resources = metadata_file.get("resources", {}) for resource_name, resource in resources.items(): if resource["type"] != "oci-image": continue - image_name = resource["upstream-source"] - logging.info(f"Downloading OCI image: {image_name}") - run(["docker", "pull", image_name]) - image_id = run( - ["docker", "image", "inspect", image_name, "--format", "'{{.Id}}'"] - ) - image_id = image_id.rstrip("\n").strip("'").removeprefix("sha256:") - assert "\n" not in image_id, f"Multiple local images found for {image_name}" - logging.info(f"Uploading charm resource: {resource_name}") - output = run( - [ + for architecture in charm_revisions.keys(): + # Format: ST111 - Multi-architecture `upstream-source` in charm OCI resources + # https://docs.google.com/document/d/19pzpza7zj7qswDRSHBlpqdBrA7Ndcnyh6_75cCxMKSo/edit + upstream_source = resource.get("upstream-source") + if upstream_source is not None and "upstream-sources" in resource: + raise ValueError( + "`upstream-sources` and `upstream-source` cannot be used simultaneously. Use only `upstream-sources`" + ) + elif upstream_source: + # Default to X64 + upstream_sources = {charmcraft.Architecture.X64.value: upstream_source} + else: + upstream_sources = resource["upstream-sources"] + image_name = upstream_sources[architecture.value] + logging.info(f"Downloading OCI image ({architecture=}): {image_name}") + run(["docker", "pull", image_name]) + image_id = run( + ["docker", "image", "inspect", image_name, "--format", "'{{.Id}}'"] + ) + image_id = image_id.rstrip("\n").strip("'").removeprefix("sha256:") + assert "\n" not in image_id, f"Multiple local images found for {image_name}" + logging.info(f"Uploading charm resource: {resource_name}") + output = run( + [ + "charmcraft", + "upload-resource", + "--format", + "json", + charm_name, + resource_name, + "--image", + image_id, + ] + ) + revision: int = json.loads(output)["revision"] + logging.info(f"Uploaded charm resource {revision=}") + oci_resources.setdefault(architecture, []).append( + OCIResource(resource_name, revision) + ) + + # Release charm file(s) + for architecture, revisions in charm_revisions.items(): + for charm_revision in revisions: + logging.info(f"Releasing {charm_revision=} {architecture=}") + command = [ "charmcraft", - "upload-resource", - "--format", - "json", + "release", charm_name, - resource_name, - "--image", - image_id, + "--revision", + str(charm_revision), + "--channel", + args.channel, ] - ) - revision: int = json.loads(output)["revision"] - logging.info(f"Uploaded charm resource {revision=}") - oci_resources.append(OCIResource(resource_name, revision)) - - # Release charm file(s) - for charm_revision in charm_revisions: - logging.info(f"Releasing {charm_revision=}") - command = [ - "charmcraft", - "release", - charm_name, - "--revision", - str(charm_revision), - "--channel", - args.channel, - ] - for oci in oci_resources: - command += ["--resource", f"{oci.resource_name}:{oci.revision}"] - run(command) + for oci in oci_resources[architecture]: + command += ["--resource", f"{oci.resource_name}:{oci.revision}"] + run(command) # Output GitHub release info - release_tag = f"rev{max(charm_revisions)}" - if len(charm_revisions) == 1: + revisions = [] + for revs in charm_revisions.values(): + revisions.extend(revs) + release_tag = f"rev{max(revisions)}" + if len(revisions) == 1: release_title = "Revision " else: release_title = "Revisions " - release_title += ", ".join(str(revision) for revision in charm_revisions) - release_notes = f"Released to {args.channel}\nOCI images:\n" + "\n".join( - f"- {dataclasses.asdict(oci)}" for oci in oci_resources - ) + release_title += ", ".join(str(revision) for revision in revisions) + oci_info = "OCI images:" + for architecture, resources in oci_resources.items(): + oci_info += f"\n- {architecture}:" + for oci in resources: + oci_info += f"\n - {dataclasses.asdict(oci)}" + release_notes = f"Released to {args.channel}\n{oci_info}" with open("release_notes.txt", "w") as file: file.write(release_notes) output = f"release_tag={release_tag}\nrelease_title={release_title}" diff --git a/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py b/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py index fcf3c227..06f7cd30 100644 --- a/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py +++ b/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py @@ -22,6 +22,7 @@ async def build_charm( self, charm_path: typing.Union[str, os.PathLike], bases_index: int = None ) -> pathlib.Path: charm_path = pathlib.Path(charm_path) + # TODO: add support for multiple architectures if bases_index is not None: charmcraft_yaml = yaml.safe_load((charm_path / "charmcraft.yaml").read_text()) assert charmcraft_yaml["type"] == "charm" @@ -31,7 +32,7 @@ async def build_charm( version = base.get("build-on", [base])[0]["channel"] packed_charms = list(charm_path.glob(f"*{version}-amd64.charm")) else: - packed_charms = list(charm_path.glob("*.charm")) + packed_charms = list(charm_path.glob("*-amd64.charm")) if len(packed_charms) == 1: # python-libjuju's model.deploy(), juju deploy, and juju bundle files expect local charms # to begin with `./` or `/` to distinguish them from Charmhub charms.