Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

tests/e2e: Add auth registry libvirt tests #1932

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

tests/e2e: Add auth registry libvirt tests #1932

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
36a8c06
tests/e2e: Various authentication related refactors
stevenhorsman Jul 25, 2024
6d9a8eb
tests/e2e: Add auth registry libvirt tests
stevenhorsman Aug 12, 2024
b1b0b5f
tests/e2e: Add auth registry docker tests
stevenhorsman Jul 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 0 additions & 42 deletions src/cloud-api-adaptor/test/e2e/assessment_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ func NewTestCase(t *testing.T, e env.Environment, testName string, assert CloudA
assessMessage: assessMessage,
podState: v1.PodRunning,
imagePullTimer: false,
isAuth: false,
deletionWithin: assert.DefaultTimeout(),
}

Expand Down Expand Up @@ -300,47 +299,6 @@ func GetSuccessfulAndErroredPods(ctx context.Context, t *testing.T, client klien
return successPod, errorPod, podLogString, nil
}

func GetAuthenticatedImageStatus(ctx context.Context, client klient.Client, expectedStatus string, authpod v1.Pod) error {
clientset, err := kubernetes.NewForConfig(client.RESTConfig())
if err != nil {
return err
}
watcher, err := clientset.CoreV1().Events(authpod.ObjectMeta.Namespace).Watch(ctx, metav1.ListOptions{})
if err != nil {
return err
}
defer watcher.Stop()
for event := range watcher.ResultChan() {
if event.Object.(*v1.Event).InvolvedObject.Name == authpod.ObjectMeta.Name {
if event.Object.(*v1.Event).Type == "Normal" && event.Object.(*v1.Event).Reason == "Started" {
return nil
}
if event.Object.(*v1.Event).Type == "Warning" && (strings.Contains(event.Object.(*v1.Event).Message, "failed to authorize") || strings.Contains(event.Object.(*v1.Event).Message, "illegal base64 data at input byte") || strings.Contains(event.Object.(*v1.Event).Message, "401 UNAUTHORIZED")) {
if expectedStatus == "Completed" {
return errors.New("Invalid Credentials: " + event.Object.(*v1.Event).Message)
} else {
return nil
}
}

if event.Object.(*v1.Event).Type == "Warning" && strings.Contains(event.Object.(*v1.Event).Message, "not found") {
return errors.New("Invalid Image Name: " + event.Object.(*v1.Event).Message)
}

if event.Object.(*v1.Event).Type == "Warning" && strings.Contains(event.Object.(*v1.Event).Message, "failed to pull manifest Not authorized") {
if expectedStatus == "Completed" {
return errors.New("Invalid auth-json-secret: " + event.Object.(*v1.Event).Message)
} else {
return nil
}
}

}
}

return errors.New("PodVM Start Error")
}

// SkipTestOnCI skips the test if running on CI
func SkipTestOnCI(t *testing.T) {
ci := os.Getenv("CI")
Expand Down
48 changes: 29 additions & 19 deletions src/cloud-api-adaptor/test/e2e/assessment_runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ type ExtraPod struct {
pod *v1.Pod
imagePullTimer bool
expectedPodLogString string
isAuth bool
testInstanceTypes InstanceValidatorFunctions
podState v1.PodPhase
testCommands []TestCommand
Expand All @@ -71,10 +70,10 @@ type TestCase struct {
service *v1.Service
testCommands []TestCommand
expectedPodLogString string
expectedPodDescribe string
podState v1.PodPhase
imagePullTimer bool
isAuth bool
AuthImageStatus string
noAuthJson bool
deletionWithin time.Duration
testInstanceTypes InstanceValidatorFunctions
isNydusSnapshotter bool
Expand Down Expand Up @@ -146,6 +145,11 @@ func (tc *TestCase) WithExpectedPodLogString(expectedPodLogString string) *TestC
return tc
}

func (tc *TestCase) WithExpectedPodDescribe(expectedPodDescribe string) *TestCase {
tc.expectedPodDescribe = expectedPodDescribe
return tc
}

func (tc *TestCase) WithCustomPodState(customPodState v1.PodPhase) *TestCase {
tc.podState = customPodState
return tc
Expand All @@ -156,13 +160,8 @@ func (tc *TestCase) WithPodWatcher() *TestCase {
return tc
}

func (tc *TestCase) WithAuthenticatedImage() *TestCase {
tc.isAuth = true
return tc
}

func (tc *TestCase) WithAuthImageStatus(status string) *TestCase {
tc.AuthImageStatus = status
func (tc *TestCase) WithNoAuthJson() *TestCase {
tc.noAuthJson = true
return tc
}

Expand Down Expand Up @@ -236,7 +235,7 @@ func (tc *TestCase) Run() {
}
}

if tc.AuthImageStatus == "WithoutCredentials" {
if tc.noAuthJson {
clientSet, err := kubernetes.NewForConfig(client.RESTConfig())
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -367,12 +366,27 @@ func (tc *TestCase) Run() {
t.Logf("Log output of peer pod:%s", LogString)
}

if tc.isAuth {
if err := GetAuthenticatedImageStatus(ctx, client, tc.AuthImageStatus, *tc.pod); err != nil {
if tc.expectedPodDescribe != "" {
if err := client.Resources(tc.pod.Namespace).List(ctx, &podlist); err != nil {
t.Fatal(err)
}

t.Logf("PodVM has successfully reached %v state with authenticated Image - %v", tc.AuthImageStatus, os.Getenv("AUTHENTICATED_REGISTRY_IMAGE"))
for _, podItem := range podlist.Items {
if podItem.ObjectMeta.Name == tc.pod.Name {
podEvent, err := PodEventExtractor(ctx, client, *tc.pod)
if err != nil {
t.Fatal(err)
}
t.Logf("podEvent: %+v\n", podEvent)
if strings.Contains(podEvent.EventDescription, tc.expectedPodDescribe) {
t.Logf("Output Log from Pod: %s", podEvent)
} else {
t.Errorf("Job Created pod with Invalid log")
}
break
} else {
t.Fatal("Pod Not Found...")
}
}
}

if tc.testInstanceTypes.testSuccessfn != nil && tc.testInstanceTypes.testFailurefn != nil {
Expand Down Expand Up @@ -508,10 +522,6 @@ func (tc *TestCase) Run() {
}
t.Logf("Log output of peer pod:%s", LogString)
}
if extraPod.isAuth {
// TBD
t.Fatal("Error: isAuth hasn't been implemented in extraPods. Please implement assess function for isAuth")
}
if extraPod.testInstanceTypes.testSuccessfn != nil && extraPod.testInstanceTypes.testFailurefn != nil {
// TBD
t.Fatal("Error: testInstanceTypes hasn't been implemented in extraPods. Please implement assess for function testInstanceTypes.")
Expand Down
43 changes: 6 additions & 37 deletions src/cloud-api-adaptor/test/e2e/common_suite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ package e2e

import (
"bytes"
"encoding/json"
"fmt"
"math/rand"
"os"
Expand Down Expand Up @@ -241,52 +240,22 @@ func DoTestCreatePeerPodWithPVCAndCSIWrapper(t *testing.T, e env.Environment, as
NewTestCase(t, e, "PeerPodWithPVCAndCSIWrapper", assert, "PVC is created and mounted as expected").WithPod(pod).WithPVC(myPVC).WithTestCommands(testCommands).Run()
}

func DoTestCreatePeerPodWithAuthenticatedImagewithValidCredentials(t *testing.T, e env.Environment, assert CloudAssert) {
func DoTestCreatePeerPodWithAuthenticatedImageWithValidCredentials(t *testing.T, e env.Environment, assert CloudAssert) {
randseed := rand.New(rand.NewSource(time.Now().UnixNano()))
podName := "authenticated-image-valid-" + strconv.Itoa(int(randseed.Uint32())) + "-pod"
expectedAuthStatus := "Completed"
podName := "authenticated-image-with-creds-" + strconv.Itoa(int(randseed.Uint32())) + "-pod"
imageName := os.Getenv("AUTHENTICATED_REGISTRY_IMAGE")
pod := NewPod(E2eNamespace, podName, podName, imageName, WithRestartPolicy(v1.RestartPolicyNever))
NewTestCase(t, e, "ValidAuthImagePeerPod", assert, "Peer pod with Authenticated Image with Valid Credentials(Default service account) has been created").WithPod(pod).WithAuthenticatedImage().WithAuthImageStatus(expectedAuthStatus).WithCustomPodState(v1.PodPending).Run()
}

func DoTestCreatePeerPodWithAuthenticatedImageWithInvalidCredentials(t *testing.T, e env.Environment, assert CloudAssert) {
registryName := "quay.io"
if os.Getenv("AUTHENTICATED_REGISTRY_IMAGE") != "" {
registryName = strings.Split(os.Getenv("AUTHENTICATED_REGISTRY_IMAGE"), "/")[0]
}
randseed := rand.New(rand.NewSource(time.Now().UnixNano()))
podName := "authenticated-image-invalid-" + strconv.Itoa(int(randseed.Uint32())) + "-pod"
secretName := "auth-json-secret-invalid"
data := map[string]interface{}{
"auths": map[string]interface{}{
registryName: map[string]interface{}{
"auth": "aW52YWxpZHVzZXJuYW1lOmludmFsaWRwYXNzd29yZAo=",
},
},
}
jsondata, err := json.MarshalIndent(data, "", " ")
if err != nil {
t.Fatal(err)
}
if err != nil {
t.Fatal(err)
}
expectedAuthStatus := "ImagePullBackOff"
secretData := map[string][]byte{v1.DockerConfigJsonKey: jsondata}
secret := NewSecret(E2eNamespace, secretName, secretData, v1.SecretTypeDockerConfigJson)
imageName := os.Getenv("AUTHENTICATED_REGISTRY_IMAGE")
pod := NewPod(E2eNamespace, podName, podName, imageName, WithRestartPolicy(v1.RestartPolicyNever), WithImagePullSecrets(secretName))
NewTestCase(t, e, "InvalidAuthImagePeerPod", assert, "Peer pod with Authenticated Image with Invalid Credentials has been created").WithSecret(secret).WithPod(pod).WithAuthenticatedImage().WithAuthImageStatus(expectedAuthStatus).WithCustomPodState(v1.PodPending).Run()
NewTestCase(t, e, "ValidAuthImagePeerPod", assert, "Peer pod with Authenticated Image with Valid Credentials(Default service account) has been created").WithPod(pod).WithCustomPodState(v1.PodRunning).Run()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @stevenhorsman !

I'm curious why originally it was checking for v1.PodPending but now v1.PodRunning (which seems the correct indeed).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can't recall the logic as I think I re-wrote it 2 months ago, but I think I ended up surprised that some of these tests ever passed. We did only run them for the ibm cloud provider, which has not had well supported tests for a long long time, so it might have been that if you got lucky with pull timing then pending was enough, but I was trying to make them more robust.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, got it. thx!

}

// Check that without creds the image can't be pulled to ensure we don't have a false positive in our auth test
func DoTestCreatePeerPodWithAuthenticatedImageWithoutCredentials(t *testing.T, e env.Environment, assert CloudAssert) {
randseed := rand.New(rand.NewSource(time.Now().UnixNano()))
podName := "authenticated-image-without-creds-" + strconv.Itoa(int(randseed.Uint32())) + "-pod"
expectedAuthStatus := "WithoutCredentials"
imageName := os.Getenv("AUTHENTICATED_REGISTRY_IMAGE")
pod := NewPod(E2eNamespace, podName, podName, imageName, WithRestartPolicy(v1.RestartPolicyNever))
NewTestCase(t, e, "InvalidAuthImagePeerPod", assert, "Peer pod with Authenticated Image without Credentials has been created").WithPod(pod).WithAuthenticatedImage().WithAuthImageStatus(expectedAuthStatus).WithCustomPodState(v1.PodPending).Run()
expectedErrorString := "401 UNAUTHORIZED"
NewTestCase(t, e, "InvalidAuthImagePeerPod", assert, "Peer pod with Authenticated Image without Credentials has been created").WithPod(pod).WithNoAuthJson().WithExpectedPodDescribe(expectedErrorString).WithCustomPodState(v1.PodPending).Run()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The generalization to allow running tests WithExpectedPodDescribe(expectedErrorString) is amazing! Thanks!

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, the framework has lots of building blocks, which can be helpful, but many of them are quite specific, which just makes it a struggle to understand the flow and which path you are going down (for me at least), so I wanted to simplify and make some more basic ones that mimic more how I test things myself.

}

func DoTestPodVMwithNoAnnotations(t *testing.T, e env.Environment, assert CloudAssert, expectedType string) {
Expand Down
19 changes: 19 additions & 0 deletions src/cloud-api-adaptor/test/e2e/docker_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package e2e

import (
"os"
"testing"

_ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/docker"
Expand Down Expand Up @@ -110,3 +111,21 @@ func TestDockerKbsKeyRelease(t *testing.T) {
keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego")
DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint)
}

func TestDockerCreatePeerPodWithAuthenticatedImageWithoutCredentials(t *testing.T) {
assert := DockerAssert{}
if os.Getenv("AUTHENTICATED_REGISTRY_IMAGE") != "" {
DoTestCreatePeerPodWithAuthenticatedImageWithoutCredentials(t, testEnv, assert)
} else {
t.Skip("Authenticated Image Name not exported")
}
}

func TestDockerCreatePeerPodWithAuthenticatedImageWithValidCredentials(t *testing.T) {
assert := DockerAssert{}
if os.Getenv("REGISTRY_CREDENTIAL_ENCODED") != "" && os.Getenv("AUTHENTICATED_REGISTRY_IMAGE") != "" {
DoTestCreatePeerPodWithAuthenticatedImageWithValidCredentials(t, testEnv, assert)
} else {
t.Skip("Registry Credentials, or authenticated image name not exported")
}
}
15 changes: 2 additions & 13 deletions src/cloud-api-adaptor/test/e2e/ibmcloud_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,23 +143,12 @@ func TestCreatePeerPodWithPVC(t *testing.T) {
}
}

func TestCreatePeerPodWithAuthenticatedImagewithValidCredentials(t *testing.T) {
func TestCreatePeerPodWithAuthenticatedImageWithValidCredentials(t *testing.T) {
assert := IBMCloudAssert{
VPC: pv.IBMCloudProps.VPC,
}
if os.Getenv("REGISTRY_CREDENTIAL_ENCODED") != "" && os.Getenv("AUTHENTICATED_REGISTRY_IMAGE") != "" {
DoTestCreatePeerPodWithAuthenticatedImagewithValidCredentials(t, testEnv, assert)
} else {
t.Skip("Registry Credentials not exported")
}
}

func TestCreatePeerPodWithAuthenticatedImageWithInvalidCredentials(t *testing.T) {
assert := IBMCloudAssert{
VPC: pv.IBMCloudProps.VPC,
}
if os.Getenv("REGISTRY_CREDENTIAL_ENCODED") != "" && os.Getenv("AUTHENTICATED_REGISTRY_IMAGE") != "" {
DoTestCreatePeerPodWithAuthenticatedImageWithInvalidCredentials(t, testEnv, assert)
DoTestCreatePeerPodWithAuthenticatedImageWithValidCredentials(t, testEnv, assert)
} else {
t.Skip("Registry Credentials not exported")
}
Expand Down
19 changes: 19 additions & 0 deletions src/cloud-api-adaptor/test/e2e/libvirt_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package e2e

import (
"os"
"testing"

_ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/libvirt"
Expand Down Expand Up @@ -139,3 +140,21 @@ func TestLibvirtPermissivePolicyAllowsExec(t *testing.T) {
assert := LibvirtAssert{}
DoTestPermissivePolicyAllowsExec(t, testEnv, assert)
}

func TestLibvirtCreatePeerPodWithAuthenticatedImageWithoutCredentials(t *testing.T) {
assert := LibvirtAssert{}
if os.Getenv("AUTHENTICATED_REGISTRY_IMAGE") != "" {
DoTestCreatePeerPodWithAuthenticatedImageWithoutCredentials(t, testEnv, assert)
} else {
t.Skip("Authenticated Image Name not exported")
}
}

func TestLibvirtCreatePeerPodWithAuthenticatedImageWithValidCredentials(t *testing.T) {
assert := LibvirtAssert{}
if os.Getenv("REGISTRY_CREDENTIAL_ENCODED") != "" && os.Getenv("AUTHENTICATED_REGISTRY_IMAGE") != "" {
DoTestCreatePeerPodWithAuthenticatedImageWithValidCredentials(t, testEnv, assert)
} else {
t.Skip("Registry Credentials, or authenticated image name not exported")
}
}
4 changes: 4 additions & 0 deletions src/cloud-api-adaptor/test/provisioner/docker/provision_common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,10 @@ func (lio *DockerInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config,
}
}

if err := lio.Overlay.SetAuthJsonSecretIfApplicable(); err != nil {
return err
}

if err := lio.Overlay.YamlReload(); err != nil {
return err
}
Expand Down
46 changes: 4 additions & 42 deletions src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,11 @@ import (
"encoding/json"
"io"
"net/http"
"os"
"path/filepath"
"strings"

pv "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner"
log "github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/e2e-framework/pkg/envconf"
)

Expand Down Expand Up @@ -167,46 +164,11 @@ func (lio *IBMCloudInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config
}
}
}
if os.Getenv("REGISTRY_CREDENTIAL_ENCODED") != "" {
registryName := "quay.io"
client, err := cfg.NewClient()
if err != nil {
return err
}
clientSet, err := kubernetes.NewForConfig(client.RESTConfig())
if err != nil {
return err
}
_, err = clientSet.CoreV1().Secrets("confidential-containers-system").Get(ctx, "auth-json-secret", metav1.GetOptions{})
if err == nil {
log.Info("Deleting pre-existing auth-json-secret...")
err = clientSet.CoreV1().Secrets("confidential-containers-system").Delete(ctx, "auth-json-secret", metav1.DeleteOptions{})
if err != nil {
return err
}
}
if os.Getenv("AUTHENTICATED_REGISTRY_IMAGE") != "" {
registryName = strings.Split(os.Getenv("AUTHENTICATED_REGISTRY_IMAGE"), "/")[0]
}
log.Info("Setting up auth.json")
data := map[string]interface{}{
"auths": map[string]interface{}{
registryName: map[string]interface{}{
"auth": os.Getenv("REGISTRY_CREDENTIAL_ENCODED"),
},
},
}
jsondata, err := json.MarshalIndent(data, "", " ")
if err != nil {
return err
}
if err := os.WriteFile(filepath.Join(lio.Overlay.ConfigDir, "auth.json"), jsondata, 0644); err != nil {
return err
}
if err = lio.Overlay.SetKustomizeSecretGeneratorFile("auth-json-secret", "auth.json"); err != nil {
return err
}

if err = lio.Overlay.SetAuthJsonSecretIfApplicable(); err != nil {
return err
}

if err = lio.Overlay.YamlReload(); err != nil {
return err
}
Expand Down
Loading
Loading