Merge pull request #1703 from jaimergp/gha-self-hosted-cirun

Add support for Cirun on self-hosted GHA runners

conda_smithy/cirun_utils.py

@@ -0,0 +1,82 @@
+"""
+See http://py.cirun.io/api.html for cirun client docs
+"""
+import os
+from functools import lru_cache
+from typing import List, Dict, Any, Optional
+
+from cirun import Cirun
+from .github import gh_token, Github
+
+
+@lru_cache
+def get_cirun_installation_id(owner: str) -> int:
+    # This ID needs a token with admin: org privileges.
+    # Hard-code instead for easier use.
+    if owner == "conda-forge":
+        return 18453316
+    else:
+        gh = Github(gh_token)
+        user = gh.get_user()
+        if user.login == owner:
+            user_or_org = user
+        else:
+            user_or_org = gh.get_organization(owner)
+        for inst in user_or_org.get_installations:
+            if inst.raw_data["app_slug"] == "cirun-application":
+                return inst.app_id
+        raise ValueError(f"cirun not found for owner {owner}")
+
+
+def enable_cirun_for_project(owner: str, repo: str) -> Dict[str, Any]:
+    """Enable the cirun.io Github Application for a particular repository."""
+    print(f"Enabling cirun for {owner}/{repo} ...")
+    cirun = _get_cirun_client()
+    return cirun.set_repo(
+        f"{owner}/{repo}", installation_id=get_cirun_installation_id(owner)
+    )
+
+
+def add_repo_to_cirun_resource(
+    owner: str,
+    repo: str,
+    resource: str,
+    cirun_policy_args: Optional[List[str]] = None,
+) -> Dict[str, Any]:
+    """Grant access to a cirun resource to a particular repository, with a particular policy."""
+    cirun = _get_cirun_client()
+    policy_args: Optional[Dict[str, Any]] = None
+    if cirun_policy_args and "pull_request" in cirun_policy_args:
+        policy_args = {"pull_request": True}
+    print(
+        f"Adding repo {owner}/{repo} to resource {resource} with policy_args: {policy_args}"
+    )
+    response = cirun.add_repo_to_resources(
+        owner,
+        repo,
+        resources=[resource],
+        teams=[repo],
+        policy_args=policy_args,
+    )
+    print(f"response: {response} | {response.json().keys()}")
+    return response
+
+
+def remove_repo_from_cirun_resource(owner: str, repo: str, resource: str):
+    """Revoke access to a cirun resource to a particular repository, with a particular policy."""
+    cirun = _get_cirun_client()
+    print(f"Removing repo {owner}/{repo} from resource {resource}.")
+    response = cirun.remove_repo_from_resources(owner, repo, [resource])
+    print(f"response: {response} | {response.json().keys()}")
+    return response
+
+
+@lru_cache
+def _get_cirun_client() -> Cirun:
+    try:
+        return Cirun()
+    except KeyError:
+        raise RuntimeError(
+            "You must have CIRUN_API_KEY defined to do Cirun CI registration. "
+            "This requirement can be overriden by specifying `--without-cirun`"
+        )

conda_smithy/cli.py

@@ -12,6 +12,8 @@
 import conda  # noqa
 import conda_build.api
 from conda_build.metadata import MetaData
+
+import conda_smithy.cirun_utils
 from conda_smithy.utils import get_feedstock_name_from_meta, merge_dict
 from ruamel.yaml import YAML
 
@@ -208,7 +210,7 @@ def __init__(self, parser):
         # conda-smithy register-ci ./
         super(RegisterCI, self).__init__(
             parser,
-            "Register a feedstock at the CI " "services which do the builds.",
+            "Register a feedstock at the CI services which do the builds.",
         )
         scp = self.subcommand_parser
         scp.add_argument(
@@ -238,6 +240,7 @@ def __init__(self, parser):
             "Appveyor",
             "Drone",
             "Webservice",
+            "Cirun",
         ]:
             scp.add_argument(
                 "--without-{}".format(ci.lower()),
@@ -257,6 +260,23 @@ def __init__(self, parser):
             action="append",
             help="drone server URL to register this repo. multiple values allowed",
         )
+        scp.add_argument(
+            "--cirun-resources",
+            default=[],
+            action="append",
+            help="cirun resources to enable for this repo. multiple values allowed",
+        )
+        scp.add_argument(
+            "--cirun-policy-args",
+            action="append",
+            help="extra arguments for cirun policy to create for this repo. multiple values allowed",
+        )
+        scp.add_argument(
+            "--remove",
+            action="store_true",
+            help="Revoke access to the configured CI services. "
+            "Only available for Cirun for now",
+        )
 
     def __call__(self, args):
         from conda_smithy import ci_register
@@ -278,7 +298,19 @@ def __call__(self, args):
             )
 
         print("CI Summary for {}/{} (can take ~30s):".format(owner, repo))
-
+        if args.remove and any(
+            [
+                args.azure,
+                args.circle,
+                args.appveyor,
+                args.drone,
+                args.webservice,
+                args.anaconda_token,
+            ]
+        ):
+            raise RuntimeError(
+                "The --remove flag is only supported for Cirun for now"
+            )
         if not args.anaconda_token:
             print(
                 "Warning: By not registering an Anaconda/Binstar token"
@@ -345,6 +377,31 @@ def __call__(self, args):
         else:
             print("Drone registration disabled.")
 
+        if args.cirun:
+            print("Cirun Registration")
+            if args.remove:
+                if args.cirun_resources:
+                    to_remove = args.cirun_resources
+                else:
+                    to_remove = ["*"]
+
+                print(f"Cirun Registration: resources to remove: {to_remove}")
+                for resource in to_remove:
+                    conda_smithy.cirun_utils.remove_repo_from_cirun_resource(
+                        owner, repo, resource
+                    )
+            else:
+                print(
+                    f"Cirun Registration: resources to add to: {owner}/{repo}"
+                )
+                conda_smithy.cirun_utils.enable_cirun_for_project(owner, repo)
+                for resource in args.cirun_resources:
+                    conda_smithy.cirun_utils.add_repo_to_cirun_resource(
+                        owner, repo, resource, args.cirun_policy_args
+                    )
+        else:
+            print("Cirun registration disabled.")
+
         if args.webservice:
             ci_register.add_conda_forge_webservice_hooks(owner, repo)
         else:

conda_smithy/configure_feedstock.py

@@ -377,6 +377,9 @@ def _collapse_subpackage_variants(
     if not is_noarch:
         always_keep_keys.add("target_platform")
 
+    if forge_config["github_actions"]["self_hosted"]:
+        always_keep_keys.add("github_actions_labels")
+
     all_used_vars.update(always_keep_keys)
     all_used_vars.update(top_level_vars)
 
@@ -695,7 +698,6 @@ def _render_ci_provider(
                 channel_target.startswith("conda-forge ")
                 and provider_name == "github_actions"
                 and not forge_config["github_actions"]["self_hosted"]
-                and os.path.basename(forge_dir) not in SERVICE_FEEDSTOCKS
             ):
                 raise RuntimeError(
                     "Using github_actions as the CI provider inside "
@@ -1266,6 +1268,82 @@ def render_appveyor(jinja_env, forge_config, forge_dir, return_metadata=False):
 def _github_actions_specific_setup(
     jinja_env, forge_config, forge_dir, platform
 ):
+    # Handle GH-hosted and self-hosted runners runs-on config
+    # Do it before the deepcopy below so these changes can be used by the
+    # .github/worfkflows/conda-build.yml template
+    runs_on = {
+        "osx-64": {
+            "os": "macos",
+            "self_hosted_labels": ("macOS", "x64"),
+        },
+        "osx-arm64": {
+            "os": "macos",
+            "self_hosted_labels": ("macOS", "arm64"),
+        },
+        "linux-64": {
+            "os": "ubuntu",
+            "self_hosted_labels": ("linux", "x64"),
+        },
+        "linux-aarch64": {
+            "os": "ubuntu",
+            "self_hosted_labels": ("linux", "ARM64"),
+        },
+        "win-64": {
+            "os": "windows",
+            "self_hosted_labels": ("windows", "x64"),
+        },
+        "win-arm64": {
+            "os": "windows",
+            "self_hosted_labels": ("windows", "ARM64"),
+        },
+    }
+    for data in forge_config["configs"]:
+        if not data["build_platform"].startswith(platform):
+            continue
+        # This Github Actions specific configs are prefixed with "gha_"
+        # because we are not deepcopying the data dict intentionally
+        # so it can be used in the general "render_github_actions" function
+        # This avoid potential collisions with other CI providers :crossed_fingers:
+        data["gha_os"] = runs_on[data["build_platform"]]["os"]
+        data["gha_with_gpu"] = False
+
+        self_hosted_default = list(
+            runs_on[data["build_platform"]]["self_hosted_labels"]
+        )
+        self_hosted_default += ["self-hosted"]
+        hosted_default = [data["gha_os"] + "-latest"]
+
+        labels_default = (
+            ["hosted"]
+            if forge_config["github_actions"]["self_hosted"]
+            else ["self-hosted"]
+        )
+        labels = conda_build.utils.ensure_list(
+            data["config"].get("github_actions_labels", [labels_default])[0]
+        )
+
+        if len(labels) == 1 and labels[0] == "hosted":
+            labels = hosted_default
+        elif len(labels) == 1 and labels[0] in "self-hosted":
+            labels = self_hosted_default
+        else:
+            # Prepend the required ones
+            labels += self_hosted_default
+
+        if forge_config["github_actions"]["self_hosted"]:
+            data["gha_runs_on"] = []
+            # labels provided in conda-forge.yml
+            for label in labels:
+                if label.startswith("cirun-"):
+                    label += (
+                        "--${{ github.run_id }}-" + data["short_config_name"]
+                    )
+                if "gpu" in label.lower():
+                    data["gha_with_gpu"] = True
+                data["gha_runs_on"].append(label)
+        else:
+            data["gha_runs_on"] = hosted_default
+
     build_setup = _get_build_setup_line(forge_dir, platform, forge_config)
 
     if platform == "linux":
@@ -1288,11 +1366,13 @@ def _github_actions_specific_setup(
             ".scripts/run_win_build.bat",
         ],
     }
-    if forge_config["github_actions"]["store_build_artifacts"]:
-        for tmpls in platform_templates.values():
-            tmpls.append(".scripts/create_conda_build_artifacts.sh")
+
     template_files = platform_templates.get(platform, [])
 
+    # Templates for all platforms
+    if forge_config["github_actions"]["store_build_artifacts"]:
+        template_files.append(".scripts/create_conda_build_artifacts.sh")
+
     _render_template_exe_files(
         forge_config=forge_config,
         jinja_env=jinja_env,
@@ -1307,7 +1387,7 @@ def render_github_actions(
     target_path = os.path.join(
         forge_dir, ".github", "workflows", "conda-build.yml"
     )
-    template_filename = "github-actions.tmpl"
+    template_filename = "github-actions.yml.tmpl"
     fast_finish_text = ""
 
     (
@@ -1317,7 +1397,7 @@ def render_github_actions(
         upload_packages,
     ) = _get_platforms_of_provider("github_actions", forge_config)
 
-    logger.debug("github platforms retreived")
+    logger.debug("github platforms retrieved")
 
     remove_file_or_dir(target_path)
     return _render_ci_provider(
@@ -1841,6 +1921,9 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None):
         },
         "github_actions": {
             "self_hosted": False,
+            "triggers": [],
+            "timeout_minutes": 360,
+            "cancel_in_progress": True,
             # Set maximum parallel jobs
             "max_parallel": None,
             # Toggle creating artifacts for conda build_artifacts dir
@@ -2001,6 +2084,12 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None):
     if config["test"] is None:
         config["test"] = "all"
 
+    if not config["github_actions"]["triggers"]:
+        self_hosted = config["github_actions"]["self_hosted"]
+        config["github_actions"]["triggers"] = (
+            ["push"] if self_hosted else ["push", "pull_request"]
+        )
+
     # An older conda-smithy used to have some files which should no longer exist,
     # remove those now.
     old_files = [