From 1e126f43e6387945851613bef3fd7e82348b2c92 Mon Sep 17 00:00:00 2001 From: Sonny Garcia Date: Thu, 10 Nov 2022 16:45:27 -0700 Subject: [PATCH] EKS Stuffz (#78) * adds basic eks workflow structure * * Added EKS functional test set up + teardown. (#79) - Adding EKS test set up * creates GenericImageBuilderTestSuite for use by multiple cloud implementations * * Updated EKS functional test set up to use the shared functions. * * Split the repo URL to access the registry and repo in ECR. * * Added parsing of the repository URL. * * Updated testenv to include kubeconfig changes. * Added filename to EKS set up. * updates testenv dep and adds aws-sdk so we can pull ecr credentials * changes cert-manager default webhook port to avoid collision with eks' default kubelet port...hurumph * fixes GenericImageBuilderTestSuite so that a new CloudAuthTest func can encapsulate the details of each cloud enviornment also makes some changes to the LoadBalancer service lookup checking for the Hostname first, then defaulting to the IP when that's blank * updates functional test workflow with complete EKS Co-authored-by: Jade Co-authored-by: Jade Hayes --- .github/workflows/functional.yml | 80 +++++ test/functional/eks_test.go | 85 ++++++ test/functional/gke_test.go | 497 +++---------------------------- test/functional/go.mod | 14 +- test/functional/go.sum | 28 +- test/functional/helmfile.yaml | 2 + test/functional/helpers_test.go | 466 ++++++++++++++++++++++++++++- 7 files changed, 712 insertions(+), 460 deletions(-) diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index f7577434..427011ec 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -5,6 +5,85 @@ on: types: [created] jobs: + eks: + if: ${{ github.event.issue.pull_request && github.event.comment.body == '/functional-test' }} + name: EKS image building + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Checkout pull request + run: hub pr checkout ${{ github.event.issue.number }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Update PR Comment + uses: peter-evans/create-or-update-comment@v2 + with: + comment-id: ${{ github.event.comment.id }} + body: | + + **Launched workflow:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + reactions: rocket + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.AWS_IAM_ROLE }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version-file: test/functional/go.mod + + - id: go-cache-paths + name: Gather Go cache paths + run: | + echo "::set-output name=go-build::$(go env GOCACHE)" + echo "::set-output name=go-mod::$(go env GOMODCACHE)" + + - name: Go build cache + uses: actions/cache@v3 + with: + path: ${{ steps.go-cache-paths.outputs.go-build }} + key: ${{ runner.os }}-go-build-${{ hashFiles('test/functional/**/*.go') }} + + - name: Go mod cache + uses: actions/cache@v3 + with: + path: ${{ steps.go-cache-paths.outputs.go-mod }} + key: ${{ runner.os }}-go-mod-${{ hashFiles('test/functional/go.sum') }} + + - name: Install Helm + uses: azure/setup-helm@v3 + with: + version: v3.10.1 + + - name: Run tests + env: + VERBOSE_TESTING: true + GCP_REGION: ${{ secrets.GCP_REGION }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + MANAGER_IMAGE_TAG: pr-${{ github.event.issue.number }} + run: | + cd test/functional + go test -timeout 0 -tags functional,eks + + - name: Save testenv files + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: testenv-eks + path: | + test/functional/testenv + !test/functional/testenv/**/.terraform/ + gke: if: ${{ github.event.issue.pull_request && github.event.comment.body == '/functional-test' }} name: GKE image building @@ -84,6 +163,7 @@ jobs: test/functional/testenv !test/functional/testenv/**/.terraform/ + aks: name: AKS image building runs-on: ubuntu-latest diff --git a/test/functional/eks_test.go b/test/functional/eks_test.go index a75bc859..8f2f46d1 100644 --- a/test/functional/eks_test.go +++ b/test/functional/eks_test.go @@ -1 +1,86 @@ +//go:build functional && eks + package functional + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ecr" + hephv1 "github.com/dominodatalab/hephaestus/pkg/api/hephaestus/v1" + "github.com/dominodatalab/testenv" + "github.com/heroku/docker-registry-client/registry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "k8s.io/utils/pointer" +) + +func TestEKSFunctionality(t *testing.T) { + suite.Run(t, new(EKSTestSuite)) +} + +type EKSTestSuite struct { + GenericImageBuilderTestSuite + + region string +} + +func (suite *EKSTestSuite) SetupSuite() { + suite.region = os.Getenv("AWS_REGION") + suite.CloudAuthTest = suite.testCloudAuth + suite.CloudConfigFunc = func() testenv.CloudConfig { + return testenv.EKSConfig{ + Region: suite.region, + KubernetesVersion: os.Getenv("KUBERNETES_VERSION"), + } + } + + suite.GenericImageBuilderTestSuite.SetupSuite() +} + +func (suite *EKSTestSuite) testCloudAuth(ctx context.Context, t *testing.T) { + fullRepo, err := suite.manager.OutputVar(ctx, "repository") + require.NoError(t, err) + + canonicalImage := string(fullRepo) + cloudRegistry := strings.SplitN(canonicalImage, "/", 2)[0] + cloudRepository := strings.SplitN(canonicalImage, "/", 2)[1] + + build := newImageBuild( + python39JupyterBuildContext, + canonicalImage, + &hephv1.RegistryCredentials{ + Server: cloudRegistry, + CloudProvided: pointer.Bool(true), + }, + ) + ib := createBuild(t, ctx, suite.hephClient, build) + + conf, err := config.LoadDefaultConfig(ctx, config.WithEC2IMDSRegion()) + require.NoError(t, err) + + client := ecr.NewFromConfig(conf) + input := &ecr.GetAuthorizationTokenInput{} + resp, err := client.GetAuthorizationToken(ctx, input) + require.NoError(t, err) + + authToken := aws.ToString(resp.AuthorizationData[0].AuthorizationToken) + decoded, err := base64.StdEncoding.DecodeString(authToken) + require.NoError(t, err) + + credentials := strings.SplitN(string(decoded), ":", 2) + + hub, err := registry.New(fmt.Sprintf("https://%s", cloudRegistry), credentials[0], credentials[1]) + require.NoError(t, err) + + tags, err := hub.Tags(cloudRepository) + require.NoError(t, err) + assert.Contains(t, tags, ib.Spec.LogKey) +} diff --git a/test/functional/gke_test.go b/test/functional/gke_test.go index e980aea1..101058a5 100644 --- a/test/functional/gke_test.go +++ b/test/functional/gke_test.go @@ -5,25 +5,16 @@ package functional import ( "context" "fmt" - "net/url" "os" - "sync" "testing" - "time" hephv1 "github.com/dominodatalab/hephaestus/pkg/api/hephaestus/v1" - "github.com/dominodatalab/hephaestus/pkg/clientset" "github.com/dominodatalab/testenv" "github.com/heroku/docker-registry-client/registry" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" auth "golang.org/x/oauth2/google" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" "k8s.io/utils/pointer" ) @@ -32,467 +23,65 @@ func TestGKEFunctionality(t *testing.T) { } type GKETestSuite struct { - suite.Suite + GenericImageBuilderTestSuite - suiteSetupDone bool - gcpRegistry string - gcpRepository string - - manager testenv.Manager - k8sClient kubernetes.Interface - hephClient clientset.Interface + region string + projectID string } func (suite *GKETestSuite) SetupSuite() { - verbose := os.Getenv("VERBOSE_TESTING") == "true" - region := os.Getenv("GCP_REGION") - projectID := os.Getenv("GCP_PROJECT_ID") - k8sVersion := os.Getenv("KUBERNETES_VERSION") - managerImageTag := os.Getenv("MANAGER_IMAGE_TAG") - - cfg := testenv.GKEConfig{ - Region: region, - ProjectID: projectID, - KubernetesVersion: k8sVersion, - KubernetesServiceAccount: "default/hephaestus", + suite.region = os.Getenv("GCP_REGION") + suite.projectID = os.Getenv("GCP_PROJECT_ID") + suite.CloudAuthTest = suite.testCloudAuth + suite.CloudConfigFunc = func() testenv.CloudConfig { + return testenv.GKEConfig{ + Region: suite.region, + ProjectID: suite.projectID, + KubernetesVersion: os.Getenv("KUBERNETES_VERSION"), + KubernetesServiceAccount: "default/hephaestus", + } } + suite.VariableFunc = func(ctx context.Context) { + gcpServiceAccount, err := suite.manager.OutputVar(ctx, "service_account") + require.NoError(suite.T(), err) - var err error - ctx := context.Background() - - suite.manager, err = testenv.NewCloudEnvManager(ctx, cfg, verbose) - require.NoError(suite.T(), err) - defer func() { - if !suite.suiteSetupDone { - suite.TearDownSuite() + suite.helmfileValues = []string{ + "controller.manager.cloudRegistryAuth.gcp.enabled=true", + fmt.Sprintf("controller.manager.cloudRegistryAuth.gcp.serviceAccount=%s", gcpServiceAccount), } - }() - - suite.T().Log("Creating GKE test environment") - start := time.Now() - require.NoError(suite.T(), suite.manager.Create(ctx)) - suite.T().Logf("Total cluster creation time: %s", time.Since(start)) - - repoName, err := suite.manager.OutputVar(ctx, "repository") - require.NoError(suite.T(), err) - suite.gcpRegistry = fmt.Sprintf("%s-docker.pkg.dev", region) - suite.gcpRepository = fmt.Sprintf("%s/%s", projectID, repoName) - - gcpServiceAccount, err := suite.manager.OutputVar(ctx, "service_account") - require.NoError(suite.T(), err) - - helmfileValues := []string{ - "controller.manager.cloudRegistryAuth.gcp.enabled=true", - fmt.Sprintf("controller.manager.cloudRegistryAuth.gcp.serviceAccount=%s", gcpServiceAccount), - fmt.Sprintf("controller.manager.image.tag=%s", managerImageTag), } - suite.T().Log("Installing cluster applications") - start = time.Now() - require.NoError(suite.T(), suite.manager.HelmfileApply(ctx, "helmfile.yaml", helmfileValues)) - suite.T().Logf("Total application install time: %s", time.Since(start)) - - configBytes, err := suite.manager.KubeconfigBytes(ctx) - require.NoError(suite.T(), err) - - clientConfig, err := clientcmd.NewClientConfigFromBytes(configBytes) - require.NoError(suite.T(), err) - - restConfig, err := clientConfig.ClientConfig() - require.NoError(suite.T(), err) - - suite.k8sClient, err = kubernetes.NewForConfig(restConfig) - require.NoError(suite.T(), err) - - suite.hephClient, err = clientset.NewForConfig(restConfig) - require.NoError(suite.T(), err) - - suite.T().Log("Test setup complete") - suite.suiteSetupDone = true -} - -func (suite *GKETestSuite) TearDownSuite() { - suite.T().Log("Tearing down test cluster") - require.NoError(suite.T(), suite.manager.Destroy(context.Background())) + suite.GenericImageBuilderTestSuite.SetupSuite() } -func (suite *GKETestSuite) TestImageBuildValidation() { - suite.T().Log("Testing image build validation") +func (suite *GKETestSuite) testCloudAuth(ctx context.Context, t *testing.T) { + repoName, err := suite.manager.OutputVar(ctx, "repository") + require.NoError(suite.T(), err) - ctx := context.Background() - client := suite.hephClient.HephaestusV1().ImageBuilds(corev1.NamespaceDefault) + cloudRegistry := fmt.Sprintf("%s-docker.pkg.dev", suite.region) + cloudRepository := fmt.Sprintf("%s/%s", suite.projectID, repoName) - tt := []struct { - name string - errContains string - mutator func(build *hephv1.ImageBuild) - }{ - { - "blank_context", - "spec.context: Required value: must not be blank", - func(build *hephv1.ImageBuild) { - build.Spec.Context = "" - }, - }, - { - "no_images", - "spec.images: Required value: must contain at least 1 image", - func(build *hephv1.ImageBuild) { - build.Spec.Images = nil - }, - }, - { - "invalid_image", - "spec.images: Invalid value: \"~cruisin' usa!!!\": invalid reference format", - func(build *hephv1.ImageBuild) { - build.Spec.Images = []string{ - "~cruisin' usa!!!", - } - }, + image := fmt.Sprintf("%s/test-image", cloudRepository) + build := newImageBuild( + python39JupyterBuildContext, + fmt.Sprintf("%s/%s", cloudRegistry, image), + &hephv1.RegistryCredentials{ + Server: cloudRegistry, + CloudProvided: pointer.Bool(true), }, - { - "bad_build_args", - "spec.buildArgs[0]: Invalid value: \"i have no equals sign\": must use a = format, " + - "spec.buildArgs[1]: Invalid value: \" =value\": must use a = format", - func(build *hephv1.ImageBuild) { - build.Spec.BuildArgs = []string{ - "i have no equals sign", - " =value", - } - }, - }, - { - "blank_auth_server", - "spec.registryAuth[0].server: Required value: must not be blank", - func(build *hephv1.ImageBuild) { - build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ - { - Server: "", - BasicAuth: &hephv1.BasicAuthCredentials{ - Username: "username", - Password: "password", - }, - }, - } - }, - }, - { - "no_auth_credential_sources", - "spec.registryAuth[0]: Required value: must specify 1 credential source", - func(build *hephv1.ImageBuild) { - build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ - { - Server: "docker-registry.default:5000", - }, - } - }, - }, - { - "multiple_auth_credential_sources", - "spec.registryAuth[0]: Forbidden: cannot specify more than 1 credential source", - func(build *hephv1.ImageBuild) { - build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ - { - Server: "docker-registry.default:5000", - BasicAuth: &hephv1.BasicAuthCredentials{ - Username: "username", - Password: "password", - }, - Secret: &hephv1.SecretCredentials{ - Name: "name", - Namespace: "namespace", - }, - }, - } - }, - }, - { - "no_username_basic_auth_credentials", - "spec.registryAuth[0].basicAuth.username: Required value: must not be blank", - func(build *hephv1.ImageBuild) { - build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ - { - Server: "docker-registry.default:5000", - BasicAuth: &hephv1.BasicAuthCredentials{ - Password: "password", - }, - }, - } - }, - }, - { - "no_password_basic_auth_credentials", - "spec.registryAuth[0].basicAuth.password: Required value: must not be blank", - func(build *hephv1.ImageBuild) { - build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ - { - Server: "docker-registry.default:5000", - BasicAuth: &hephv1.BasicAuthCredentials{ - Username: "username", - }, - }, - } - }, - }, - { - "no_name_secret_credentials", - "spec.registryAuth[0].secret.name: Required value: must not be blank", - func(build *hephv1.ImageBuild) { - build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ - { - Server: "docker-registry.default:5000", - Secret: &hephv1.SecretCredentials{ - Namespace: "namespace", - }, - }, - } - }, - }, - { - "no_namespace_secret_credentials", - "spec.registryAuth[0].secret.namespace: Required value: must not be blank", - func(build *hephv1.ImageBuild) { - build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ - { - Server: "docker-registry.default:5000", - Secret: &hephv1.SecretCredentials{ - Name: "name", - }, - }, - } - }, - }, - } + ) + ib := createBuild(t, ctx, suite.hephClient, build) - for _, tc := range tt { - suite.T().Logf("Test case: %s", tc.name) - suite.T().Run(tc.name, func(t *testing.T) { - build := &hephv1.ImageBuild{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-build-", - }, - Spec: hephv1.ImageBuildSpec{ - Context: "https://nowhere.com/docker-build-context.tgz", - Images: []string{ - "registry/org/repo:tag", - }, - }, - } - tc.mutator(build) + credentials, err := auth.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/cloud-platform") + require.NoError(t, err) - var statusErr *apierrors.StatusError - _, err := client.Create(ctx, build, metav1.CreateOptions{}) - require.ErrorAs(t, err, &statusErr) + token, err := credentials.TokenSource.Token() + require.NoError(t, err) - errStatus := statusErr.ErrStatus + hub, err := registry.New(fmt.Sprintf("https://%s", cloudRegistry), "oauth2accesstoken", token.AccessToken) + require.NoError(t, err) - assert.Equal(t, metav1.StatusFailure, errStatus.Status) - assert.Equal(t, metav1.StatusReasonInvalid, errStatus.Reason) - assert.Contains(t, errStatus.Message, tc.errContains) - }) - } -} - -func (suite *GKETestSuite) TestImageBuilding() { - ctx := context.Background() - - suite.T().Run("no_auth", func(t *testing.T) { - build := newImageBuild( - python39JupyterBuildContext, - "docker-registry:5000/test-ns/test-repo", - nil, - ) - ib := createBuild(t, ctx, suite.hephClient, build) - require.NotEqual(t, hephv1.PhaseFailed, ib.Status.Phase) - - svc, err := suite.k8sClient.CoreV1().Services(corev1.NamespaceDefault).Get( - ctx, - "docker-registry", - metav1.GetOptions{}, - ) - require.NoError(t, err) - - registryURL, err := url.Parse(fmt.Sprintf("http://%s:%d", svc.Status.LoadBalancer.Ingress[0].IP, 5000)) - require.NoError(t, err) - - hub, err := registry.New(registryURL.String(), "", "") - require.NoError(t, err) - - tags, err := hub.Tags("test-ns/test-repo") - require.NoError(t, err) - assert.Contains(t, tags, ib.Spec.LogKey) - - testLogDelivery(t, ctx, suite.k8sClient, ib) - testMessageDelivery(t, ctx, suite.k8sClient, ib) - }) - - suite.T().Run("bad_auth", func(t *testing.T) { - build := newImageBuild( - python39JupyterBuildContext, - "docker-registry-secure:5000/test-ns/test-repo", - &hephv1.RegistryCredentials{ - Server: "docker-registry-secure:5000", - BasicAuth: &hephv1.BasicAuthCredentials{ - Username: "bad", - Password: "stuff", - }, - }, - ) - ib := createBuild(t, ctx, suite.hephClient, build) - - assert.Equal(t, ib.Status.Phase, hephv1.PhaseFailed) - assert.Contains(t, ib.Status.Conditions[0].Message, `"docker-registry-secure:5000" client credentials are invalid`) - }) - - suite.T().Run("basic_auth", func(t *testing.T) { - build := newImageBuild( - python39JupyterBuildContext, - "docker-registry-secure:5000/test-ns/test-repo", - &hephv1.RegistryCredentials{ - Server: "docker-registry-secure:5000", - BasicAuth: &hephv1.BasicAuthCredentials{ - Username: "test-user", - Password: "test-password", - }, - }, - ) - ib := createBuild(t, ctx, suite.hephClient, build) - - assert.Equalf(t, ib.Status.Phase, hephv1.PhaseSucceeded, "failed build with message %q", ib.Status.Conditions[0].Message) - }) - - suite.T().Run("secret_auth", func(t *testing.T) { - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-secret-", - }, - Type: corev1.SecretTypeDockerConfigJson, - StringData: map[string]string{ - corev1.DockerConfigJsonKey: `{"auths":{"docker-registry-secure:5000":{"username":"test-user","password":"test-password"}}}`, - }, - } - secretClient := suite.k8sClient.CoreV1().Secrets(corev1.NamespaceDefault) - secret, err := secretClient.Create(ctx, secret, metav1.CreateOptions{}) - require.NoError(t, err, "failed to create docker credentials secret") - defer secretClient.Delete(ctx, secret.Name, metav1.DeleteOptions{}) - - build := newImageBuild( - python39JupyterBuildContext, - "docker-registry-secure:5000/test-ns/test-repo", - &hephv1.RegistryCredentials{ - Server: "docker-registry-secure:5000", - Secret: &hephv1.SecretCredentials{ - Name: secret.Name, - Namespace: secret.Namespace, - }, - }, - ) - ib := createBuild(t, ctx, suite.hephClient, build) - - assert.Equalf(t, ib.Status.Phase, hephv1.PhaseSucceeded, "failed build with message %q", ib.Status.Conditions[0].Message) - }) - - suite.T().Run("cloud_auth", func(t *testing.T) { - image := fmt.Sprintf("%s/test-image", suite.gcpRepository) - build := newImageBuild( - python39JupyterBuildContext, - fmt.Sprintf("%s/%s", suite.gcpRegistry, image), - &hephv1.RegistryCredentials{ - Server: suite.gcpRegistry, - CloudProvided: pointer.Bool(true), - }, - ) - ib := createBuild(t, ctx, suite.hephClient, build) - - credentials, err := auth.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/cloud-platform") - require.NoError(t, err) - - token, err := credentials.TokenSource.Token() - require.NoError(t, err) - - hub, err := registry.New(fmt.Sprintf("https://%s", suite.gcpRegistry), "oauth2accesstoken", token.AccessToken) - require.NoError(t, err) - - tags, err := hub.Tags(image) - require.NoError(t, err) - assert.Contains(t, tags, ib.Spec.LogKey) - }) - - suite.T().Run("build_args", func(t *testing.T) { - build := newImageBuild( - buildArgBuildContext, - "docker-registry:5000/test-ns/test-repo", - nil, - ) - build.Spec.BuildArgs = []string{"INPUT=VAR=VAL"} - ib := createBuild(t, ctx, suite.hephClient, build) - - assert.Equalf(t, ib.Status.Phase, hephv1.PhaseSucceeded, "failed build with message %q", ib.Status.Conditions[0].Message) - }) - - suite.T().Run("build_failure", func(t *testing.T) { - build := newImageBuild( - errorBuildContext, - "docker-registry:5000/test-ns/test-repo", - nil, - ) - ib := createBuild(t, ctx, suite.hephClient, build) - - assert.Equalf(t, ib.Status.Phase, hephv1.PhaseFailed, "expected build with bad Dockerfile to fail") - }) - - suite.T().Run("multi_stage", func(t *testing.T) { - build := newImageBuild( - multiStageBuildContext, - "docker-registry:5000/test-ns/test-repo", - nil, - ) - ib := createBuild(t, ctx, suite.hephClient, build) - - assert.Equalf(t, ib.Status.Phase, hephv1.PhaseSucceeded, "failed build with message %q", ib.Status.Conditions[0].Message) - }) - - suite.T().Run("concurrent_builds", func(t *testing.T) { - var wg sync.WaitGroup - ch := make(chan *hephv1.ImageBuild, 3) - - for i := 0; i < 3; i++ { - wg.Add(1) - - go func() { - defer wg.Done() - - build := newImageBuild( - dseBuildContext, - "docker-registry:5000/test-ns/test-repo", - nil, - ) - build.Spec.DisableLocalBuildCache = true - - ch <- createBuild(t, ctx, suite.hephClient, build) - }() - } - wg.Wait() - close(ch) - - var builders []string - for ib := range ch { - builders = append(builders, ib.Status.BuilderAddr) - assert.Equalf( - t, - ib.Status.Phase, - hephv1.PhaseSucceeded, - "failed build %q with message %q", - ib.Name, - ib.Status.Conditions[0].Message, - ) - } - - expected := []string{ - "tcp://hephaestus-buildkit-0.hephaestus-buildkit.default:1234", - "tcp://hephaestus-buildkit-1.hephaestus-buildkit.default:1234", - "tcp://hephaestus-buildkit-2.hephaestus-buildkit.default:1234", - } - assert.ElementsMatch(t, builders, expected, "builds did not execute on unique buildkit pods") - }) + tags, err := hub.Tags(image) + require.NoError(t, err) + assert.Contains(t, tags, ib.Spec.LogKey) } diff --git a/test/functional/go.mod b/test/functional/go.mod index 04876298..a7f52319 100644 --- a/test/functional/go.mod +++ b/test/functional/go.mod @@ -5,8 +5,11 @@ go 1.19 //replace github.com/dominodatalab/testenv => ../../../testenv require ( + github.com/aws/aws-sdk-go-v2 v1.11.2 + github.com/aws/aws-sdk-go-v2/config v1.11.1 + github.com/aws/aws-sdk-go-v2/service/ecr v1.12.0 github.com/dominodatalab/hephaestus v0.1.21 - github.com/dominodatalab/testenv v0.0.0-20221102225811-ae1286c64f8c + github.com/dominodatalab/testenv v0.0.0-20221110225415-2dc11cfd48f5 github.com/go-playground/validator/v10 v10.11.1 github.com/go-redis/redis/v9 v9.0.0-rc.1 github.com/heroku/docker-registry-client v0.0.0-20211012143308-9463674c8930 @@ -50,6 +53,15 @@ require ( github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/avast/retry-go/v4 v4.0.2 // indirect github.com/aws/aws-sdk-go v1.40.28 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.6.5 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.7.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.12.0 // indirect + github.com/aws/smithy-go v1.9.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/blang/semver v3.5.1+incompatible // indirect diff --git a/test/functional/go.sum b/test/functional/go.sum index 8f42fdf7..777f0f57 100644 --- a/test/functional/go.sum +++ b/test/functional/go.sum @@ -177,6 +177,30 @@ github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi github.com/aws/aws-sdk-go v1.37.18/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.28 h1:IWzkX36BHx9R4jYd5y8NAudk8sxUeJHHohZgPI9kq/A= github.com/aws/aws-sdk-go v1.40.28/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go-v2 v1.11.2 h1:SDiCYqxdIYi6HgQfAWRhgdZrdnOuGyLDJVRSWLeHWvs= +github.com/aws/aws-sdk-go-v2 v1.11.2/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= +github.com/aws/aws-sdk-go-v2/config v1.11.1 h1:KXSjb7ZMLRtjxClFptukTYibiOqJS9NwBO+9WD3UMto= +github.com/aws/aws-sdk-go-v2/config v1.11.1/go.mod h1:VvfkzUhVtntSg1JfGFMSKS0CyiTZd3NqBxK5af4zsME= +github.com/aws/aws-sdk-go-v2/credentials v1.6.5 h1:ZrsO2js2v4T95rsCIWoAb/ck5+U1kwkizGdZHY+ni3s= +github.com/aws/aws-sdk-go-v2/credentials v1.6.5/go.mod h1:HWSOnsnqVMbLcWUmom6AN1cqhcLzLJ62AObW28CbYbU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 h1:KiN5TPOLrEjbGCvdTQR4t0U4T87vVwALZ5Bg3jpMqPY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2/go.mod h1:dF2F6tXEOgmW5X1ZFO/EPtWrcm7XkW07KNcJUGNtt4s= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 h1:XJLnluKuUxQG255zPNe+04izXl7GSyUVafIsgfv9aw4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2/go.mod h1:SgKKNBIoDC/E1ZCDhhMW3yalWjwuLjMcpLzsM/QQnWo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 h1:EauRoYZVNPlidZSZJDscjJBQ22JhVF2+tdteatax2Ak= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2/go.mod h1:xT4XX6w5Sa3dhg50JrYyy3e4WPYo/+WjY/BXtqXVunU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 h1:IQup8Q6lorXeiA/rK72PeToWoWK8h7VAPgHNWdSrtgE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2/go.mod h1:VITe/MdW6EMXPb0o0txu/fsonXbMHUU2OC2Qp7ivU4o= +github.com/aws/aws-sdk-go-v2/service/ecr v1.12.0 h1:oDIFK9jio/g88kjEihtkxt2IKelm7LjE75hX4x7rxoU= +github.com/aws/aws-sdk-go-v2/service/ecr v1.12.0/go.mod h1:IoE3h7WVE1zmlQzUHEYJ5JtfrF4g3rCG8mPz+fsp0+s= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 h1:CKdUNKmuilw/KNmO2Q53Av8u+ZyXMC2M9aX8Z+c/gzg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2/go.mod h1:FgR1tCsn8C6+Hf+N5qkfrE4IXvUL1RgW87sunJ+5J4I= +github.com/aws/aws-sdk-go-v2/service/sso v1.7.0 h1:E4fxAg/UE8a6yiLZYv8/EP0uXKPPRImiMau4ift6S/g= +github.com/aws/aws-sdk-go-v2/service/sso v1.7.0/go.mod h1:KnIpszaIdwI33tmc/W/GGXyn22c1USYxA/2KyvoeDY0= +github.com/aws/aws-sdk-go-v2/service/sts v1.12.0 h1:7g0252k2TF3eA1DtfkTQB/tqI41YvbUPaolwTR0/ITc= +github.com/aws/aws-sdk-go-v2/service/sts v1.12.0/go.mod h1:UV2N5HaPfdbDpkgkz4sRzWCvQswZjdO1FfqCWl0t7RA= +github.com/aws/smithy-go v1.9.0 h1:c7FUdEqrQA1/UVKKCNDFQPNKGp4FQg3YW4Ck5SLTG58= +github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -256,8 +280,8 @@ github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNE github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dominodatalab/hephaestus v0.1.21 h1:3fNkyjxf8OxoViCIJskpqOejXTRVLzp50zEKngZnVX8= github.com/dominodatalab/hephaestus v0.1.21/go.mod h1:AVGa+u58L+G8iD+RPgPvQP2OAipDWL3tsqt6Tofo158= -github.com/dominodatalab/testenv v0.0.0-20221102225811-ae1286c64f8c h1:lnOcxf085qWsJ5jyt0XmHojUY3nLBv+TqS2iTMDZfX8= -github.com/dominodatalab/testenv v0.0.0-20221102225811-ae1286c64f8c/go.mod h1:S6ua68CDPVyaj249bsb4X+ObGr6zjiv23m767hHdFKQ= +github.com/dominodatalab/testenv v0.0.0-20221110225415-2dc11cfd48f5 h1:eYqGrjFcsc9rtbm/lxVKuQ1LOb+68CblCwN/4G99n1k= +github.com/dominodatalab/testenv v0.0.0-20221110225415-2dc11cfd48f5/go.mod h1:S6ua68CDPVyaj249bsb4X+ObGr6zjiv23m767hHdFKQ= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= diff --git a/test/functional/helmfile.yaml b/test/functional/helmfile.yaml index b0cff586..ad1bc763 100644 --- a/test/functional/helmfile.yaml +++ b/test/functional/helmfile.yaml @@ -45,6 +45,8 @@ releases: - installCRDs: true extraArgs: - --enable-certificate-owner-ref + webhook: + securePort: 10260 - name: docker-registry namespace: default diff --git a/test/functional/helpers_test.go b/test/functional/helpers_test.go index 42fe5049..2ea6bd56 100644 --- a/test/functional/helpers_test.go +++ b/test/functional/helpers_test.go @@ -5,23 +5,475 @@ import ( "encoding/json" "fmt" "math/rand" + "net/url" + "os" + "sync" "testing" "time" hephv1 "github.com/dominodatalab/hephaestus/pkg/api/hephaestus/v1" "github.com/dominodatalab/hephaestus/pkg/clientset" "github.com/dominodatalab/hephaestus/pkg/messaging/amqp" + "github.com/dominodatalab/testenv" "github.com/go-playground/validator/v10" "github.com/go-redis/redis/v9" + "github.com/heroku/docker-registry-client/registry" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" ) +type GenericImageBuilderTestSuite struct { + suite.Suite + + CloudAuthTest func(context.Context, *testing.T) + CloudConfigFunc func() testenv.CloudConfig + VariableFunc func(context.Context) + + manager testenv.Manager + hephClient clientset.Interface + k8sClient kubernetes.Interface + + helmfileValues []string + suiteSetupDone bool +} + +func (suite *GenericImageBuilderTestSuite) SetupSuite() { + ctx := context.Background() + verbose := os.Getenv("VERBOSE_TESTING") == "true" + + if suite.CloudConfigFunc == nil { + suite.T().Fatal("CloudConfigFunc is nil") + } + config := suite.CloudConfigFunc() + + var err error + suite.manager, err = testenv.NewCloudEnvManager(ctx, config, verbose) + require.NoError(suite.T(), err) + defer func() { + if !suite.suiteSetupDone { + suite.TearDownSuite() + } + }() + + suite.T().Log("Creating test environment") + start := time.Now() + require.NoError(suite.T(), suite.manager.Create(ctx)) + suite.T().Logf("Total cluster creation time: %s", time.Since(start)) + + if suite.VariableFunc != nil { + suite.VariableFunc(ctx) + } + + if managerImageTag, ok := os.LookupEnv("MANAGER_IMAGE_TAG"); ok { + suite.helmfileValues = append(suite.helmfileValues, fmt.Sprintf("controller.manager.image.tag=%s", managerImageTag)) + } + + suite.T().Log("Installing cluster applications") + start = time.Now() + require.NoError(suite.T(), suite.manager.HelmfileApply(ctx, "helmfile.yaml", suite.helmfileValues)) + suite.T().Logf("Total application install time: %s", time.Since(start)) + + configBytes, err := suite.manager.KubeconfigBytes(ctx) + require.NoError(suite.T(), err) + + clientConfig, err := clientcmd.NewClientConfigFromBytes(configBytes) + require.NoError(suite.T(), err) + + restConfig, err := clientConfig.ClientConfig() + require.NoError(suite.T(), err) + + suite.k8sClient, err = kubernetes.NewForConfig(restConfig) + require.NoError(suite.T(), err) + + suite.hephClient, err = clientset.NewForConfig(restConfig) + require.NoError(suite.T(), err) + + suite.T().Log("Test setup complete") + suite.suiteSetupDone = true +} + +func (suite *GenericImageBuilderTestSuite) TearDownSuite() { + suite.T().Log("Tearing down test cluster") + // require.NoError(suite.T(), suite.manager.Destroy(context.Background())) +} + +func (suite *GenericImageBuilderTestSuite) TestImageBuildResourceValidation() { + suite.T().Log("Testing image build validation") + + ctx := context.Background() + client := suite.hephClient.HephaestusV1().ImageBuilds(corev1.NamespaceDefault) + + tt := []struct { + name string + errContains string + mutator func(build *hephv1.ImageBuild) + }{ + { + "blank_context", + "spec.context: Required value: must not be blank", + func(build *hephv1.ImageBuild) { + build.Spec.Context = "" + }, + }, + { + "no_images", + "spec.images: Required value: must contain at least 1 image", + func(build *hephv1.ImageBuild) { + build.Spec.Images = nil + }, + }, + { + "invalid_image", + "spec.images: Invalid value: \"~cruisin' usa!!!\": invalid reference format", + func(build *hephv1.ImageBuild) { + build.Spec.Images = []string{ + "~cruisin' usa!!!", + } + }, + }, + { + "bad_build_args", + "spec.buildArgs[0]: Invalid value: \"i have no equals sign\": must use a = format, " + + "spec.buildArgs[1]: Invalid value: \" =value\": must use a = format", + func(build *hephv1.ImageBuild) { + build.Spec.BuildArgs = []string{ + "i have no equals sign", + " =value", + } + }, + }, + { + "blank_auth_server", + "spec.registryAuth[0].server: Required value: must not be blank", + func(build *hephv1.ImageBuild) { + build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ + { + Server: "", + BasicAuth: &hephv1.BasicAuthCredentials{ + Username: "username", + Password: "password", + }, + }, + } + }, + }, + { + "no_auth_credential_sources", + "spec.registryAuth[0]: Required value: must specify 1 credential source", + func(build *hephv1.ImageBuild) { + build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ + { + Server: "docker-registry.default:5000", + }, + } + }, + }, + { + "multiple_auth_credential_sources", + "spec.registryAuth[0]: Forbidden: cannot specify more than 1 credential source", + func(build *hephv1.ImageBuild) { + build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ + { + Server: "docker-registry.default:5000", + BasicAuth: &hephv1.BasicAuthCredentials{ + Username: "username", + Password: "password", + }, + Secret: &hephv1.SecretCredentials{ + Name: "name", + Namespace: "namespace", + }, + }, + } + }, + }, + { + "no_username_basic_auth_credentials", + "spec.registryAuth[0].basicAuth.username: Required value: must not be blank", + func(build *hephv1.ImageBuild) { + build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ + { + Server: "docker-registry.default:5000", + BasicAuth: &hephv1.BasicAuthCredentials{ + Password: "password", + }, + }, + } + }, + }, + { + "no_password_basic_auth_credentials", + "spec.registryAuth[0].basicAuth.password: Required value: must not be blank", + func(build *hephv1.ImageBuild) { + build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ + { + Server: "docker-registry.default:5000", + BasicAuth: &hephv1.BasicAuthCredentials{ + Username: "username", + }, + }, + } + }, + }, + { + "no_name_secret_credentials", + "spec.registryAuth[0].secret.name: Required value: must not be blank", + func(build *hephv1.ImageBuild) { + build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ + { + Server: "docker-registry.default:5000", + Secret: &hephv1.SecretCredentials{ + Namespace: "namespace", + }, + }, + } + }, + }, + { + "no_namespace_secret_credentials", + "spec.registryAuth[0].secret.namespace: Required value: must not be blank", + func(build *hephv1.ImageBuild) { + build.Spec.RegistryAuth = []hephv1.RegistryCredentials{ + { + Server: "docker-registry.default:5000", + Secret: &hephv1.SecretCredentials{ + Name: "name", + }, + }, + } + }, + }, + } + + for _, tc := range tt { + suite.T().Logf("Test case: %s", tc.name) + suite.T().Run(tc.name, func(t *testing.T) { + build := &hephv1.ImageBuild{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-build-", + }, + Spec: hephv1.ImageBuildSpec{ + Context: "https://nowhere.com/docker-build-context.tgz", + Images: []string{ + "registry/org/repo:tag", + }, + }, + } + tc.mutator(build) + + var statusErr *apierrors.StatusError + _, err := client.Create(ctx, build, metav1.CreateOptions{}) + require.ErrorAs(t, err, &statusErr) + + errStatus := statusErr.ErrStatus + + assert.Equal(t, metav1.StatusFailure, errStatus.Status) + assert.Equal(t, metav1.StatusReasonInvalid, errStatus.Reason) + assert.Contains(t, errStatus.Message, tc.errContains) + }) + } +} + +func (suite *GenericImageBuilderTestSuite) TestImageBuilding() { + ctx := context.Background() + + suite.T().Run("no_auth", func(t *testing.T) { + build := newImageBuild( + python39JupyterBuildContext, + "docker-registry:5000/test-ns/test-repo", + nil, + ) + ib := createBuild(t, ctx, suite.hephClient, build) + require.NotEqual(t, hephv1.PhaseFailed, ib.Status.Phase) + + svc, err := suite.k8sClient.CoreV1().Services(corev1.NamespaceDefault).Get( + ctx, + "docker-registry", + metav1.GetOptions{}, + ) + require.NoError(t, err) + + hostname := svc.Status.LoadBalancer.Ingress[0].Hostname + if hostname == "" { + hostname = svc.Status.LoadBalancer.Ingress[0].IP + } + registryURL, err := url.Parse(fmt.Sprintf("http://%s:%d", hostname, 5000)) + require.NoError(t, err) + + hub, err := registry.New(registryURL.String(), "", "") + require.NoError(t, err) + + tags, err := hub.Tags("test-ns/test-repo") + require.NoError(t, err) + assert.Contains(t, tags, ib.Spec.LogKey) + + testLogDelivery(t, ctx, suite.k8sClient, ib) + testMessageDelivery(t, ctx, suite.k8sClient, ib) + }) + + suite.T().Run("bad_auth", func(t *testing.T) { + build := newImageBuild( + python39JupyterBuildContext, + "docker-registry-secure:5000/test-ns/test-repo", + &hephv1.RegistryCredentials{ + Server: "docker-registry-secure:5000", + BasicAuth: &hephv1.BasicAuthCredentials{ + Username: "bad", + Password: "stuff", + }, + }, + ) + ib := createBuild(t, ctx, suite.hephClient, build) + + assert.Equal(t, ib.Status.Phase, hephv1.PhaseFailed) + assert.Contains(t, ib.Status.Conditions[0].Message, `"docker-registry-secure:5000" client credentials are invalid`) + }) + + suite.T().Run("basic_auth", func(t *testing.T) { + build := newImageBuild( + python39JupyterBuildContext, + "docker-registry-secure:5000/test-ns/test-repo", + &hephv1.RegistryCredentials{ + Server: "docker-registry-secure:5000", + BasicAuth: &hephv1.BasicAuthCredentials{ + Username: "test-user", + Password: "test-password", + }, + }, + ) + ib := createBuild(t, ctx, suite.hephClient, build) + + assert.Equalf(t, ib.Status.Phase, hephv1.PhaseSucceeded, "failed build with message %q", ib.Status.Conditions[0].Message) + }) + + suite.T().Run("secret_auth", func(t *testing.T) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-secret-", + }, + Type: corev1.SecretTypeDockerConfigJson, + StringData: map[string]string{ + corev1.DockerConfigJsonKey: `{"auths":{"docker-registry-secure:5000":{"username":"test-user","password":"test-password"}}}`, + }, + } + secretClient := suite.k8sClient.CoreV1().Secrets(corev1.NamespaceDefault) + secret, err := secretClient.Create(ctx, secret, metav1.CreateOptions{}) + require.NoError(t, err, "failed to create docker credentials secret") + defer secretClient.Delete(ctx, secret.Name, metav1.DeleteOptions{}) + + build := newImageBuild( + python39JupyterBuildContext, + "docker-registry-secure:5000/test-ns/test-repo", + &hephv1.RegistryCredentials{ + Server: "docker-registry-secure:5000", + Secret: &hephv1.SecretCredentials{ + Name: secret.Name, + Namespace: secret.Namespace, + }, + }, + ) + ib := createBuild(t, ctx, suite.hephClient, build) + + assert.Equalf(t, ib.Status.Phase, hephv1.PhaseSucceeded, "failed build with message %q", ib.Status.Conditions[0].Message) + }) + + suite.T().Run("cloud_auth", func(t *testing.T) { + if suite.CloudAuthTest == nil { + t.Skip("cloud auth test not configured") + } + + suite.CloudAuthTest(ctx, t) + }) + + suite.T().Run("build_args", func(t *testing.T) { + build := newImageBuild( + buildArgBuildContext, + "docker-registry:5000/test-ns/test-repo", + nil, + ) + build.Spec.BuildArgs = []string{"INPUT=VAR=VAL"} + ib := createBuild(t, ctx, suite.hephClient, build) + + assert.Equalf(t, ib.Status.Phase, hephv1.PhaseSucceeded, "failed build with message %q", ib.Status.Conditions[0].Message) + }) + + suite.T().Run("build_failure", func(t *testing.T) { + build := newImageBuild( + errorBuildContext, + "docker-registry:5000/test-ns/test-repo", + nil, + ) + ib := createBuild(t, ctx, suite.hephClient, build) + + assert.Equalf(t, ib.Status.Phase, hephv1.PhaseFailed, "expected build with bad Dockerfile to fail") + }) + + suite.T().Run("multi_stage", func(t *testing.T) { + build := newImageBuild( + multiStageBuildContext, + "docker-registry:5000/test-ns/test-repo", + nil, + ) + ib := createBuild(t, ctx, suite.hephClient, build) + + assert.Equalf(t, ib.Status.Phase, hephv1.PhaseSucceeded, "failed build with message %q", ib.Status.Conditions[0].Message) + }) + + suite.T().Run("concurrent_builds", func(t *testing.T) { + t.Skip("figure out a way to ensure that the builds are actually running concurrently") + + var wg sync.WaitGroup + ch := make(chan *hephv1.ImageBuild, 3) + + for i := 0; i < 3; i++ { + wg.Add(1) + + go func() { + defer wg.Done() + + build := newImageBuild( + dseBuildContext, + "docker-registry:5000/test-ns/test-repo", + nil, + ) + build.Spec.DisableLocalBuildCache = true + + ch <- createBuild(t, ctx, suite.hephClient, build) + }() + } + wg.Wait() + close(ch) + + var builders []string + for ib := range ch { + builders = append(builders, ib.Status.BuilderAddr) + assert.Equalf( + t, + ib.Status.Phase, + hephv1.PhaseSucceeded, + "failed build %q with message %q", + ib.Name, + ib.Status.Conditions[0].Message, + ) + } + + expected := []string{ + "tcp://hephaestus-buildkit-0.hephaestus-buildkit.default:1234", + "tcp://hephaestus-buildkit-1.hephaestus-buildkit.default:1234", + "tcp://hephaestus-buildkit-2.hephaestus-buildkit.default:1234", + } + assert.ElementsMatch(t, builders, expected, "builds did not execute on unique buildkit pods") + }) +} + var seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" @@ -37,7 +489,7 @@ func randomString(length int) string { type remoteDockerBuildContext int const ( - contextServer = "https://raw.githubusercontent.com/dominodatalab/hephaestus/complete-gke-testing/test/functional/testdata/docker-context/%s/archive.tgz" + contextServer = "https://raw.githubusercontent.com/dominodatalab/hephaestus/main/test/functional/testdata/docker-context/%s/archive.tgz" buildArgBuildContext remoteDockerBuildContext = iota dseBuildContext @@ -133,8 +585,12 @@ func testLogDelivery(t *testing.T, ctx context.Context, client kubernetes.Interf ) require.NoError(t, err) + hostname := svc.Status.LoadBalancer.Ingress[0].Hostname + if hostname == "" { + hostname = svc.Status.LoadBalancer.Ingress[0].IP + } rdb := redis.NewClient(&redis.Options{ - Addr: fmt.Sprintf("%s:6379", svc.Status.LoadBalancer.Ingress[0].IP), + Addr: fmt.Sprintf("%s:6379", hostname), Password: "redis-password", }) @@ -169,7 +625,11 @@ func testMessageDelivery(t *testing.T, ctx context.Context, client kubernetes.In ) require.NoError(t, err, "failed to get rabbitmq service") - rmqURL := fmt.Sprintf("amqp://user:rabbitmq-password@%s:5672/", svc.Status.LoadBalancer.Ingress[0].IP) + hostname := svc.Status.LoadBalancer.Ingress[0].Hostname + if hostname == "" { + hostname = svc.Status.LoadBalancer.Ingress[0].IP + } + rmqURL := fmt.Sprintf("amqp://user:rabbitmq-password@%s:5672/", hostname) conn, channel, err := amqp.Dial(rmqURL) require.NoError(t, err, "failed to connet to rabbitmq service") defer func() {