Merge pull request #1833 from Nordix/fix-ironic-bmo-upgrade-1.7/adil

🐛 Fixing bmo version for clusterctl tests

scripts/ci-e2e.sh

@@ -44,8 +44,8 @@ export IMAGE_OS=${IMAGE_OS}
 export FORCE_REPO_UPDATE="false"
 EOF
 if [[ ${GINKGO_FOCUS:-} == "features" ]]; then
-    mkdir -p "$CAPI_CONFIG_FOLDER"
-    echo "enableBMHNameBasedPreallocation: true" >"$CAPI_CONFIG_FOLDER/clusterctl.yaml"
+  mkdir -p "$CAPI_CONFIG_FOLDER"
+  echo "enableBMHNameBasedPreallocation: true" >"$CAPI_CONFIG_FOLDER/clusterctl.yaml"
 fi
 # Run make devenv to boot the source cluster
 pushd "${M3_DEV_ENV_PATH}" || exit 1
@@ -118,5 +118,71 @@ export CONTRACT_TO="v1beta1"
 # image for live iso testing
 export LIVE_ISO_IMAGE="https://artifactory.nordix.org/artifactory/metal3/images/iso/minimal_linux_live-v2.iso"
 
+# Generate credentials
+BMO_OVERLAYS=(
+  "${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-0.4"
+  "${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-0.5"
+  "${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-0.6"
+)
+IRONIC_OVERLAYS=(
+  "${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-23.1"
+  "${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-24.0"
+  "${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-24.1"
+)
+
+# Create usernames and passwords and other files related to ironi basic auth if they
+# are missing
+if [[ "${IRONIC_BASIC_AUTH}" == "true" ]]; then
+  IRONIC_AUTH_DIR="${IRONIC_AUTH_DIR:-${IRONIC_DATA_DIR}/auth}"
+  mkdir -p "${IRONIC_AUTH_DIR}"
+
+  # If usernames and passwords are unset, read them from file or generate them
+  if [[ -z "${IRONIC_USERNAME:-}" ]]; then
+    if [[ ! -f "${IRONIC_AUTH_DIR}/ironic-username" ]]; then
+      IRONIC_USERNAME="$(uuid-gen)"
+      echo "${IRONIC_USERNAME}" > "${IRONIC_AUTH_DIR}/ironic-username"
+    else
+      IRONIC_USERNAME="$(cat "${IRONIC_AUTH_DIR}/ironic-username")"
+    fi
+  fi
+  if [[ -z "${IRONIC_PASSWORD:-}" ]]; then
+    if [ ! -f "${IRONIC_AUTH_DIR}/ironic-password" ]; then
+      IRONIC_PASSWORD="$(uuid-gen)"
+      echo "${IRONIC_PASSWORD}" > "${IRONIC_AUTH_DIR}/ironic-password"
+    else
+      IRONIC_PASSWORD="$(cat "${IRONIC_AUTH_DIR}/ironic-password")"
+    fi
+  fi
+  IRONIC_INSPECTOR_USERNAME="${IRONIC_INSPECTOR_USERNAME:-${IRONIC_USERNAME}}"
+  IRONIC_INSPECTOR_PASSWORD="${IRONIC_INSPECTOR_PASSWORD:-${IRONIC_PASSWORD}}"
+
+  export IRONIC_USERNAME
+  export IRONIC_PASSWORD
+  export IRONIC_INSPECTOR_USERNAME
+  export IRONIC_INSPECTOR_PASSWORD
+fi
+
+for overlay in "${BMO_OVERLAYS[@]}"; do
+  echo "${IRONIC_USERNAME}" > "${overlay}/ironic-username"
+  echo "${IRONIC_PASSWORD}" > "${overlay}/ironic-password"
+  if [[ "${overlay}" =~ release-0\.[1-5]$ ]]; then
+    echo "${IRONIC_INSPECTOR_USERNAME}" > "${overlay}/ironic-inspector-username"
+    echo "${IRONIC_INSPECTOR_PASSWORD}" > "${overlay}/ironic-inspector-password"
+  fi
+done
+
+for overlay in "${IRONIC_OVERLAYS[@]}"; do
+  echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWORD}")" > \
+    "${overlay}/ironic-htpasswd"
+  envsubst < "${REPO_ROOT}/test/e2e/data/ironic-deployment/components/basic-auth/ironic-auth-config-tpl" > \
+  "${overlay}/ironic-auth-config"
+  IRONIC_INSPECTOR_AUTH_CONFIG_TPL="/tmp/ironic-inspector-auth-config-tpl"
+  curl -o "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" https://raw.githubusercontent.com/metal3-io/baremetal-operator/release-0.5/ironic-deployment/components/basic-auth/ironic-inspector-auth-config-tpl
+  envsubst < "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" > \
+    "${overlay}/ironic-inspector-auth-config"
+  echo "INSPECTOR_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_INSPECTOR_USERNAME}" \
+    "${IRONIC_INSPECTOR_PASSWORD}")" > "${overlay}/ironic-inspector-htpasswd"
+done
+
 # run e2e tests
 make e2e-tests

test/e2e/common.go

@@ -12,18 +12,21 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 	"text/tabwriter"
 	"time"
 
+	"github.com/blang/semver"
 	bmov1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
 	infrav1 "github.com/metal3-io/cluster-api-provider-metal3/api/v1beta1"
 	ipamv1 "github.com/metal3-io/ip-address-manager/api/v1alpha1"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
 	"golang.org/x/crypto/ssh"
+	v1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,8 +38,11 @@ import (
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	testexec "sigs.k8s.io/cluster-api/test/framework/exec"
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/kustomize/api/filesys"
+	"sigs.k8s.io/kustomize/api/krusty"
 )
 
 type vmState string
@@ -793,3 +799,200 @@ func GetCAPM3StableReleaseOfMinor(ctx context.Context, minorRelease string) (str
 	releaseMarker := fmt.Sprintf(releaseMarkerPrefix, minorRelease)
 	return clusterctl.ResolveRelease(ctx, releaseMarker)
 }
+
+// GetLatestPatchRelease returns latest patch release against minor release.
+func GetLatestPatchRelease(goProxyPath string, minorReleaseVersion string) (string, error) {
+	if strings.EqualFold("main", minorReleaseVersion) || strings.EqualFold("latest", minorReleaseVersion) {
+		return strings.ToUpper(minorReleaseVersion), nil
+	}
+	minorReleaseVersion += ".0"
+	semVersion, err := semver.Parse(minorReleaseVersion)
+	if err != nil {
+		return "", errors.Wrapf(err, "parsing semver for %s", minorReleaseVersion)
+	}
+	parsedTags, err := getVersions(goProxyPath)
+	if err != nil {
+		return "", err
+	}
+
+	var picked semver.Version
+	for i, tag := range parsedTags {
+		if tag.Major == semVersion.Major && tag.Minor == semVersion.Minor {
+			picked = parsedTags[i]
+		}
+	}
+	if picked.Major == 0 && picked.Minor == 0 && picked.Patch == 0 {
+		return "", errors.Errorf("no suitable release available for path %s and version %s", goProxyPath, minorReleaseVersion)
+	}
+	return picked.String(), nil
+}
+
+// GetVersions returns the a sorted list of semantical versions which exist for a go module.
+func getVersions(gomodulePath string) (semver.Versions, error) {
+	// Get the data
+	/* #nosec G107 */
+	resp, err := http.Get(gomodulePath) //nolint:noctx
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("failed to get versions from url %s got %d %s", gomodulePath, resp.StatusCode, http.StatusText(resp.StatusCode))
+	}
+	defer resp.Body.Close()
+
+	rawResponse, err := io.ReadAll(resp.Body)
+	if err != nil {
+		retryError := errors.Wrap(err, "failed to get versions: error reading goproxy response body")
+		return nil, retryError
+	}
+	parsedVersions := semver.Versions{}
+	for _, s := range strings.Split(string(rawResponse), "\n") {
+		if s == "" {
+			continue
+		}
+		s = strings.TrimSuffix(s, "+incompatible")
+		parsedVersion, err := semver.ParseTolerant(s)
+		if err != nil {
+			// Discard releases with tags that are not a valid semantic versions (the user can point explicitly to such releases).
+			continue
+		}
+		parsedVersions = append(parsedVersions, parsedVersion)
+	}
+
+	if len(parsedVersions) == 0 {
+		return nil, fmt.Errorf("no versions found for go module %q", gomodulePath)
+	}
+	sort.Sort(parsedVersions)
+	return parsedVersions, nil
+}
+
+// BuildAndApplyKustomizationInput provides input for BuildAndApplyKustomize().
+// If WaitForDeployment and/or WatchDeploymentLogs is set to true, then DeploymentName
+// and DeploymentNamespace are expected.
+type BuildAndApplyKustomizationInput struct {
+	// Path to the kustomization to build
+	Kustomization string
+
+	ClusterProxy framework.ClusterProxy
+
+	// If this is set to true. Perform a wait until the deployment specified by
+	// DeploymentName and DeploymentNamespace is available or WaitIntervals is timed out
+	WaitForDeployment bool
+
+	// If this is set to true. Set up a log watcher for the deployment specified by
+	// DeploymentName and DeploymentNamespace
+	WatchDeploymentLogs bool
+
+	// DeploymentName and DeploymentNamespace specified a deployment that will be waited and/or logged
+	DeploymentName      string
+	DeploymentNamespace string
+
+	// Path to store the deployment logs
+	LogPath string
+
+	// Intervals to use in checking and waiting for the deployment
+	WaitIntervals []interface{}
+}
+
+func (input *BuildAndApplyKustomizationInput) validate() error {
+	// If neither WaitForDeployment nor WatchDeploymentLogs is true, we don't need to validate the input
+	if !input.WaitForDeployment && !input.WatchDeploymentLogs {
+		return nil
+	}
+	if input.WaitForDeployment && input.WaitIntervals == nil {
+		return errors.Errorf("WaitIntervals is expected if WaitForDeployment is set to true")
+	}
+	if input.WatchDeploymentLogs && input.LogPath == "" {
+		return errors.Errorf("LogPath is expected if WatchDeploymentLogs is set to true")
+	}
+	if input.DeploymentName == "" || input.DeploymentNamespace == "" {
+		return errors.Errorf("DeploymentName and DeploymentNamespace are expected if WaitForDeployment or WatchDeploymentLogs is true")
+	}
+	return nil
+}
+
+// BuildAndApplyKustomization takes input from BuildAndApplyKustomizationInput. It builds the provided kustomization
+// and apply it to the cluster provided by clusterProxy.
+func BuildAndApplyKustomization(ctx context.Context, input *BuildAndApplyKustomizationInput) error {
+	Expect(input.validate()).To(Succeed())
+	var err error
+	kustomization := input.Kustomization
+	clusterProxy := input.ClusterProxy
+	manifest, err := buildKustomizeManifest(kustomization)
+	if err != nil {
+		return err
+	}
+
+	err = clusterProxy.Apply(ctx, manifest)
+	if err != nil {
+		return err
+	}
+
+	if !input.WaitForDeployment && !input.WatchDeploymentLogs {
+		return nil
+	}
+
+	deployment := &v1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      input.DeploymentName,
+			Namespace: input.DeploymentNamespace,
+		},
+	}
+
+	if input.WaitForDeployment {
+		// Wait for the deployment to become available
+		framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
+			Getter:     clusterProxy.GetClient(),
+			Deployment: deployment,
+		}, input.WaitIntervals...)
+	}
+
+	if input.WatchDeploymentLogs {
+		// Set up log watcher
+		framework.WatchDeploymentLogsByName(ctx, framework.WatchDeploymentLogsByNameInput{
+			GetLister:  clusterProxy.GetClient(),
+			Cache:      clusterProxy.GetCache(ctx),
+			ClientSet:  clusterProxy.GetClientSet(),
+			Deployment: deployment,
+			LogPath:    input.LogPath,
+		})
+	}
+	return nil
+}
+
+// BuildAndRemoveKustomization builds the provided kustomization to resources and removes them from the cluster
+// provided by clusterProxy.
+func BuildAndRemoveKustomization(ctx context.Context, kustomization string, clusterProxy framework.ClusterProxy) error {
+	manifest, err := buildKustomizeManifest(kustomization)
+	if err != nil {
+		return err
+	}
+	return KubectlDelete(ctx, clusterProxy.GetKubeconfigPath(), manifest)
+}
+
+// KubectlDelete shells out to kubectl delete.
+func KubectlDelete(ctx context.Context, kubeconfigPath string, resources []byte, args ...string) error {
+	aargs := append([]string{"delete", "--kubeconfig", kubeconfigPath, "-f", "-"}, args...)
+	rbytes := bytes.NewReader(resources)
+	deleteCmd := testexec.NewCommand(
+		testexec.WithCommand("kubectl"),
+		testexec.WithArgs(aargs...),
+		testexec.WithStdin(rbytes),
+	)
+
+	fmt.Printf("Running kubectl %s\n", strings.Join(aargs, " "))
+	stdout, stderr, err := deleteCmd.Run(ctx)
+	fmt.Printf("stderr:\n%s\n", string(stderr))
+	fmt.Printf("stdout:\n%s\n", string(stdout))
+	return err
+}
+
+func buildKustomizeManifest(source string) ([]byte, error) {
+	kustomizer := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
+	fSys := filesys.MakeFsOnDisk()
+	resources, err := kustomizer.Run(fSys, source)
+	if err != nil {
+		return nil, err
+	}
+	return resources.AsYaml()
+}

test/e2e/config/e2e_conf.yaml

@@ -166,12 +166,20 @@ variables:
   IMAGE_CHECKSUM_TYPE: "sha256"
   IMAGE_USERNAME: "metal3"
   NODE_DRAIN_TIMEOUT: "0s"
+  IRONIC_RELEASE_23.1: "data/ironic-deployment/overlays/release-23.1"
+  IRONIC_RELEASE_24.0: "data/ironic-deployment/overlays/release-24.0"
+  IRONIC_RELEASE_24.1: "data/ironic-deployment/overlays/release-24.1"
+  IRONIC_RELEASE_LATEST: "data/ironic-deployment/overlays/release-latest"
+  BMO_RELEASE_0.4: "data/bmo-deployment/overlays/release-0.4"
+  BMO_RELEASE_0.5: "data/bmo-deployment/overlays/release-0.5"
+  BMO_RELEASE_0.6: "data/bmo-deployment/overlays/release-0.6"
+  BMO_RELEASE_LATEST: "data/bmo-deployment/overlays/release-latest"
 
 intervals:
-  default/wait-controllers: ["5m", "10s"]
+  default/wait-controllers: ["10m", "10s"]
   default/wait-cluster: ["20m", "30s"] # The second time to check the availibility of the cluster should happen late, so kcp object has time to be created
   default/wait-control-plane: ["30m", "10s"]
-  default/wait-worker-nodes: ["30m", "10s"]
+  default/wait-worker-nodes: ["60m", "10s"]
   default/wait-delete-cluster: ["20m", "10s"]
   default/wait-machine-upgrade: ["50m", "10s"]
   default/wait-machine-remediation: ["30m", "10s"]

test/e2e/data/bmo-deployment/components/basic-auth/credentials_patch.yaml

@@ -0,0 +1,18 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        volumeMounts:
+        - name: ironic-credentials
+          mountPath: "/opt/metal3/auth/ironic"
+          readOnly: true
+      volumes:
+      - name: ironic-credentials
+        secret:
+          secretName: ironic-credentials

test/e2e/data/bmo-deployment/components/basic-auth/kustomization.yaml

@@ -0,0 +1,19 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+# NOTE: This component requires a secret with the basic auth credentials!
+# How you create it is up to you. The required secrets is:
+# - ironic-credentials
+#
+# It should contain 2 fields: username and password. Example:
+#
+# apiVersion: v1
+# kind: Secret
+# metadata:
+#   name: ironic-credentials
+# data:
+#   password: <base64-encoded-password>
+#   username: <base64-encoded-username>
+
+patches:
+- path: credentials_patch.yaml

test/e2e/data/bmo-deployment/components/tls/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+patches:
+- path: tls_ca_patch.yaml
+  target:
+    kind: Deployment
+    name: controller-manager