Skip to content

Commit

Permalink
tests: add helm upgrade tests
Browse files Browse the repository at this point in the history
  • Loading branch information
pmalek committed Apr 26, 2024
1 parent c9e6845 commit f148e27
Show file tree
Hide file tree
Showing 9 changed files with 331 additions and 30 deletions.
31 changes: 31 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -31,26 +31,56 @@ require (
)

require (
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/avast/retry-go/v4 v4.5.1 // indirect
github.com/aws/aws-sdk-go v1.49.13 // indirect
github.com/bombsimon/logrusr/v3 v3.1.0 // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/distribution/reference v0.5.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sql-driver/mysql v1.7.1 // indirect
github.com/gonvenience/bunt v1.3.5 // indirect
github.com/gonvenience/neat v1.3.12 // indirect
github.com/gonvenience/term v1.0.2 // indirect
github.com/gonvenience/text v1.0.7 // indirect
github.com/gonvenience/wrap v1.1.2 // indirect
github.com/gonvenience/ytbx v1.4.4 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-github/v48 v48.2.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/gruntwork-io/go-commons v0.8.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/homeport/dyff v1.6.0 // indirect
github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect
github.com/kong/go-kong v0.54.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-ps v1.0.0 // indirect
github.com/mitchellh/hashstructure v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pquerna/otp v1.2.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/tidwall/gjson v1.17.1 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/urfave/cli v1.22.14 // indirect
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
Expand Down Expand Up @@ -119,6 +149,7 @@ require (
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-cmp v0.6.0
github.com/google/gofuzz v1.2.0 // indirect
github.com/gruntwork-io/terratest v0.46.13
github.com/imdario/mergo v0.3.16 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
Expand Down
92 changes: 89 additions & 3 deletions go.sum

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion modules/cli/cli_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func TestParse(t *testing.T) {
"--metrics-bind-address=:18080",
},
envVars: map[string]string{
"GATEWAY_OPERATOR_METRIC_BIND_ADDRESS": ":28080",
"GATEWAY_OPERATOR_METRICS_BIND_ADDRESS": ":28080",
"GATEWAY_OPERATOR_HEALTH_PROBE_BIND_ADDRESS": ":28081",
},
expectedCfg: func() manager.Config {
Expand Down
74 changes: 53 additions & 21 deletions test/e2e/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ type TestEnvironment struct {
type TestEnvOption func(opt *testEnvOptions)

type testEnvOptions struct {
Image string
Image string
InstallViaKustomize bool
}

// WithOperatorImage allows configuring the operator image to use in the test environment.
Expand All @@ -101,9 +102,18 @@ func WithOperatorImage(image string) TestEnvOption {
}
}

// WithInstallViaKustomize makes the test environment install the operator and all the
// dependencies via kustomize.
func WithInstallViaKustomize() TestEnvOption {
return func(opts *testEnvOptions) {
opts.InstallViaKustomize = true
}
}

var loggerOnce sync.Once

// CreateEnvironment creates a new independent testing environment for running isolated e2e test.
// When running with helm caller is responsible for cleaning up the environment.
func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) TestEnvironment {
t.Helper()
var opt testEnvOptions
Expand Down Expand Up @@ -154,12 +164,21 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption)
if len(opt.Image) == 0 {
opt.Image = getOperatorImage(t)
}
kustomizeDir := PrepareKustomizeDir(t, opt.Image)

var kustomizeDir KustomizeDir
if opt.InstallViaKustomize {
kustomizeDir = PrepareKustomizeDir(t, opt.Image)
}

env, err := builder.Build(ctx)
require.NoError(t, err)

t.Cleanup(func() {
cleanupEnvironment(t, context.Background(), env, kustomizeDir.Tests())
if opt.InstallViaKustomize {
cleanupEnvironment(t, context.Background(), env, kustomizeDir.Tests())
} else { //nolint:revive,staticcheck
// TODO: using helm for installation, don't clean up because we do not know the release name.
}
})

t.Logf("waiting for cluster %s and all addons to become ready", env.Cluster().Name())
Expand Down Expand Up @@ -193,28 +212,32 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption)
require.NoError(t, operatorv1alpha1.AddToScheme(clients.MgrClient.Scheme()))
require.NoError(t, operatorv1beta1.AddToScheme(clients.MgrClient.Scheme()))

t.Logf("deploying Gateway APIs CRDs from %s", testutils.GatewayExperimentalCRDsKustomizeURL)
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), testutils.GatewayExperimentalCRDsKustomizeURL))
if opt.InstallViaKustomize {
t.Logf("deploying Gateway APIs CRDs from %s", testutils.GatewayExperimentalCRDsKustomizeURL)
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), testutils.GatewayExperimentalCRDsKustomizeURL))

kicCRDsKustomizeURL := getCRDsKustomizeURLForKIC(t, versions.DefaultControlPlaneVersion)
t.Logf("deploying KIC CRDs from %s", kicCRDsKustomizeURL)
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kicCRDsKustomizeURL))
kicCRDsKustomizeURL := getCRDsKustomizeURLForKIC(t, versions.DefaultControlPlaneVersion)
t.Logf("deploying KIC CRDs from %s", kicCRDsKustomizeURL)
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kicCRDsKustomizeURL))

t.Log("creating system namespaces and serviceaccounts")
require.NoError(t, clusters.CreateNamespace(ctx, env.Cluster(), "kong-system"))
t.Log("creating system namespaces and serviceaccounts")
require.NoError(t, clusters.CreateNamespace(ctx, env.Cluster(), "kong-system"))

t.Log("deploying operator CRDs to test cluster via kustomize")
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.CRD(), "--server-side"))
t.Log("deploying operator CRDs to test cluster via kustomize")
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.CRD(), "--server-side"))

t.Log("deploying operator to test cluster via kustomize")
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.Tests(), "--server-side"))
t.Log("deploying operator to test cluster via kustomize")
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.Tests(), "--server-side"))

t.Log("waiting for operator deployment to complete")
require.NoError(t, waitForOperatorDeployment(ctx, clients.K8sClient))
t.Log("waiting for operator deployment to complete")
require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", clients.K8sClient))

t.Log("waiting for operator webhook service to be connective")
require.Eventually(t, waitForOperatorWebhookEventually(t, ctx, clients.K8sClient),
webhookReadinessTimeout, webhookReadinessTick)
t.Log("waiting for operator webhook service to be connective")
require.Eventually(t, waitForOperatorWebhookEventually(t, ctx, clients.K8sClient),
webhookReadinessTimeout, webhookReadinessTick)
} else {
t.Log("not deploying operator to test cluster via kustomize")
}

t.Log("environment is ready, starting tests")

Expand Down Expand Up @@ -269,17 +292,26 @@ func deploymentAssertConditions(conds ...appsv1.DeploymentCondition) deploymentA
}
}

func waitForOperatorDeployment(ctx context.Context, k8sClient *kubernetes.Clientset, opts ...deploymentAssertOptions) error {
func waitForOperatorDeployment(ctx context.Context, ns string, k8sClient *kubernetes.Clientset, opts ...deploymentAssertOptions) error {
outer:
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
deployment, err := k8sClient.AppsV1().Deployments("kong-system").Get(ctx, "gateway-operator-controller-manager", metav1.GetOptions{})
listOpts := metav1.ListOptions{
LabelSelector: "app.kubernetes.io/name=gateway-operator",
}
deploymentList, err := k8sClient.AppsV1().Deployments(ns).List(ctx, listOpts)
if err != nil {
return err
}
if len(deploymentList.Items) == 0 {
continue
}

deployment := &deploymentList.Items[0]

if deployment.Status.AvailableReplicas <= 0 {
continue
}
Expand Down
146 changes: 146 additions & 0 deletions test/e2e/test_helm_install_upgrade.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
package e2e

import (
"context"
"fmt"
"strings"
"testing"

"github.com/gruntwork-io/terratest/modules/helm"
"github.com/gruntwork-io/terratest/modules/k8s"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"

"github.com/kong/gateway-operator/pkg/utils/test"
)

func init() {
addTestsToTestSuite(TestUpgrade)
}

func TestUpgrade(t *testing.T) {
const (
// Rel: https://github.com/Kong/charts/tree/main/charts/gateway-operator
chart = "kong/gateway-operator"
image = "docker.io/kong/gateway-operator-oss"
)

ctx, cancel := context.WithCancel(context.Background())
defer cancel()

// createEnvironment will queue up environment cleanup if necessary
// and dumping diagnostics if the test fails.
e := CreateEnvironment(t, ctx)

testcases := []struct {
name string
fromVersion string
toVersion string
upgradeToCurrent bool
assertions func(*testing.T, *test.K8sClients)
}{
// NOTE: We do not support versions earlier than 1.2 with the helm chart.
// The initial version of the chart contained CRDs from KGO 1.2. which
// introduced a breaking change which makes it impossible to upgrade from
// automatically (without manually deleting the CRDs).
{
name: "upgrade from 1.2.0 to 1.2.3",
fromVersion: "1.2.1",
toVersion: "1.2.3",
assertions: func(t *testing.T, c *test.K8sClients) {
// TODO
},
},
{
name: "upgrade from 1.2.3 to current",
fromVersion: "1.2.3",
upgradeToCurrent: true,
},
}

var currentTag string
if imageLoad != "" {
t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD set to %q", imageLoad)
currentTag = vFromImage(t, imageLoad)
} else if imageOverride != "" {
t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE set to %q", imageOverride)
currentTag = vFromImage(t, imageOverride)
}

for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
var tag string
if tc.upgradeToCurrent {
if currentTag == "" {
t.Skipf("No KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE nor KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD" +
" env specified. Please specify the image to upgrade to in order to run this test.")
}
tag = currentTag
} else {
tag = tc.toVersion
}

tagInReleaseName := tag
if len(tag) > 8 {
tagInReleaseName = tag[:8]
}
releaseName := strings.ReplaceAll(fmt.Sprintf("kgo-%s-to-%s", tc.fromVersion, tagInReleaseName), ".", "-")
values := map[string]string{
"image.tag": tc.fromVersion,
"image.repository": image,
}

opts := &helm.Options{
KubectlOptions: &k8s.KubectlOptions{
Namespace: e.Namespace.Name,
RestConfig: e.Environment.Cluster().Config(),
},
SetValues: values,
}

require.NoError(t, helm.AddRepoE(t, opts, "kong", "https://charts.konghq.com"))
require.NoError(t, helm.InstallE(t, opts, chart, releaseName))
t.Cleanup(func() {
out, err := helm.RunHelmCommandAndGetOutputE(t, opts, "uninstall", releaseName)
if !assert.NoError(t, err) {
t.Logf("output: %s", out)
}
})

require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient,
deploymentAssertConditions(deploymentReadyConditions()...),
))

opts.SetValues["image.tag"] = tag

require.NoError(t, helm.UpgradeE(t, opts, chart, releaseName))
require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient,
deploymentAssertConditions(deploymentReadyConditions()...),
))
})
}
}

func deploymentReadyConditions() []appsv1.DeploymentCondition {
return []appsv1.DeploymentCondition{
{
Reason: "NewReplicaSetAvailable",
Status: "True",
Type: "Progressing",
},
{
Reason: "MinimumReplicasAvailable",
Status: "True",
Type: "Available",
},
}
}

func vFromImage(t *testing.T, image string) string {
splitImage := strings.Split(image, ":")
if len(splitImage) != 2 {
t.Fatalf("image %q does not contain a tag", image)
}
return splitImage[1]
}
2 changes: 1 addition & 1 deletion test/e2e/test_operator_logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func TestOperatorLogs(t *testing.T) {

// createEnvironment will queue up environment cleanup if necessary
// and dumping diagnostics if the test fails.
e := CreateEnvironment(t, ctx)
e := CreateEnvironment(t, ctx, WithInstallViaKustomize())
clients, testNamespace, cleaner := e.Clients, e.Namespace, e.Cleaner

t.Log("finding the Pod for the Gateway Operator")
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/test_upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,13 +76,13 @@ func testManifestsUpgrade(
ctx context.Context,
testParams upgradeTestParams,
) {
e := CreateEnvironment(t, ctx, WithOperatorImage(testParams.fromImage))
e := CreateEnvironment(t, ctx, WithOperatorImage(testParams.fromImage), WithInstallViaKustomize())

kustomizationDir := PrepareKustomizeDir(t, testParams.toImage)
t.Logf("deploying operator %q to test cluster %q via kustomize", testParams.toImage, e.Environment.Name())
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, e.Environment.Cluster(), kustomizationDir.Tests(), "--server-side", "-v5"))
t.Log("waiting for operator deployment to complete")
require.NoError(t, waitForOperatorDeployment(ctx, e.Clients.K8sClient,
require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", e.Clients.K8sClient,
deploymentAssertConditions(
appsv1.DeploymentCondition{
Reason: "NewReplicaSetAvailable",
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/test_webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func TestDataPlaneValidatingWebhook(t *testing.T) {

// createEnvironment will queue up environment cleanup if necessary
// and dumping diagnostics if the test fails.
e := CreateEnvironment(t, ctx)
e := CreateEnvironment(t, ctx, WithInstallViaKustomize())
clients, testNamespace := e.Clients, e.Namespace

testCases := []struct {
Expand Down
Loading

0 comments on commit f148e27

Please sign in to comment.