Skip to content

Commit

Permalink
tests: add helm upgrade tests
Browse files Browse the repository at this point in the history
  • Loading branch information
pmalek committed Apr 18, 2024
1 parent 5a434d8 commit 7972254
Show file tree
Hide file tree
Showing 9 changed files with 318 additions and 30 deletions.
31 changes: 31 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -26,26 +26,56 @@ require (
)

require (
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/avast/retry-go/v4 v4.5.1 // indirect
github.com/aws/aws-sdk-go v1.49.13 // indirect
github.com/bombsimon/logrusr/v3 v3.1.0 // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/distribution/reference v0.5.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sql-driver/mysql v1.7.1 // indirect
github.com/gonvenience/bunt v1.3.5 // indirect
github.com/gonvenience/neat v1.3.12 // indirect
github.com/gonvenience/term v1.0.2 // indirect
github.com/gonvenience/text v1.0.7 // indirect
github.com/gonvenience/wrap v1.1.2 // indirect
github.com/gonvenience/ytbx v1.4.4 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-github/v48 v48.2.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/gruntwork-io/go-commons v0.8.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/homeport/dyff v1.6.0 // indirect
github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect
github.com/kong/go-kong v0.54.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-ps v1.0.0 // indirect
github.com/mitchellh/hashstructure v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pquerna/otp v1.2.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/tidwall/gjson v1.17.1 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/urfave/cli v1.22.14 // indirect
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
Expand Down Expand Up @@ -113,6 +143,7 @@ require (
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-cmp v0.6.0
github.com/google/gofuzz v1.2.0 // indirect
github.com/gruntwork-io/terratest v0.46.13
github.com/imdario/mergo v0.3.16 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
Expand Down
92 changes: 89 additions & 3 deletions go.sum

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion modules/cli/cli_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func TestParse(t *testing.T) {
"--metrics-bind-address=:18080",
},
envVars: map[string]string{
"GATEWAY_OPERATOR_METRIC_BIND_ADDRESS": ":28080",
"GATEWAY_OPERATOR_METRICS_BIND_ADDRESS": ":28080",
"GATEWAY_OPERATOR_HEALTH_PROBE_BIND_ADDRESS": ":28081",
},
expectedCfg: func() manager.Config {
Expand Down
73 changes: 52 additions & 21 deletions test/e2e/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ type TestEnvironment struct {
type TestEnvOption func(opt *testEnvOptions)

type testEnvOptions struct {
Image string
Image string
InstallViaKustomize bool
}

// WithOperatorImage allows configuring the operator image to use in the test environment.
Expand All @@ -101,6 +102,14 @@ func WithOperatorImage(image string) TestEnvOption {
}
}

// WithInstallViaKustomize makes the test environment install the operator and all the
// dependencies via kustomize.
func WithInstallViaKustomize() TestEnvOption {
return func(opts *testEnvOptions) {
opts.InstallViaKustomize = true
}
}

var loggerOnce sync.Once

// CreateEnvironment creates a new independent testing environment for running isolated e2e test.
Expand Down Expand Up @@ -154,12 +163,21 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption)
if len(opt.Image) == 0 {
opt.Image = getOperatorImage(t)
}
kustomizeDir := PrepareKustomizeDir(t, opt.Image)

var kustomizeDir KustomizeDir
if opt.InstallViaKustomize {
kustomizeDir = PrepareKustomizeDir(t, opt.Image)
}

env, err := builder.Build(ctx)
require.NoError(t, err)

t.Cleanup(func() {
cleanupEnvironment(t, context.Background(), env, kustomizeDir.Tests())
if opt.InstallViaKustomize {
cleanupEnvironment(t, context.Background(), env, kustomizeDir.Tests())
} else {

Check failure on line 178 in test/e2e/environment.go

View workflow job for this annotation

GitHub Actions / lint

empty-block: this block is empty, you can remove it (revive)
// TODO: probably we're using helm for installation, so don't clean up.
}
})

t.Logf("waiting for cluster %s and all addons to become ready", env.Cluster().Name())
Expand Down Expand Up @@ -193,28 +211,32 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption)
require.NoError(t, operatorv1alpha1.AddToScheme(clients.MgrClient.Scheme()))
require.NoError(t, operatorv1beta1.AddToScheme(clients.MgrClient.Scheme()))

t.Logf("deploying Gateway APIs CRDs from %s", testutils.GatewayExperimentalCRDsKustomizeURL)
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), testutils.GatewayExperimentalCRDsKustomizeURL))
if opt.InstallViaKustomize {
t.Logf("deploying Gateway APIs CRDs from %s", testutils.GatewayExperimentalCRDsKustomizeURL)
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), testutils.GatewayExperimentalCRDsKustomizeURL))

kicCRDsKustomizeURL := getCRDsKustomizeURLForKIC(t, versions.DefaultControlPlaneVersion)
t.Logf("deploying KIC CRDs from %s", kicCRDsKustomizeURL)
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kicCRDsKustomizeURL))
kicCRDsKustomizeURL := getCRDsKustomizeURLForKIC(t, versions.DefaultControlPlaneVersion)
t.Logf("deploying KIC CRDs from %s", kicCRDsKustomizeURL)
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kicCRDsKustomizeURL))

t.Log("creating system namespaces and serviceaccounts")
require.NoError(t, clusters.CreateNamespace(ctx, env.Cluster(), "kong-system"))
t.Log("creating system namespaces and serviceaccounts")
require.NoError(t, clusters.CreateNamespace(ctx, env.Cluster(), "kong-system"))

t.Log("deploying operator CRDs to test cluster via kustomize")
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.CRD(), "--server-side"))
t.Log("deploying operator CRDs to test cluster via kustomize")
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.CRD(), "--server-side"))

t.Log("deploying operator to test cluster via kustomize")
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.Tests(), "--server-side"))
t.Log("deploying operator to test cluster via kustomize")
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), kustomizeDir.Tests(), "--server-side"))

t.Log("waiting for operator deployment to complete")
require.NoError(t, waitForOperatorDeployment(ctx, clients.K8sClient))
t.Log("waiting for operator deployment to complete")
require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", clients.K8sClient))

t.Log("waiting for operator webhook service to be connective")
require.Eventually(t, waitForOperatorWebhookEventually(t, ctx, clients.K8sClient),
webhookReadinessTimeout, webhookReadinessTick)
t.Log("waiting for operator webhook service to be connective")
require.Eventually(t, waitForOperatorWebhookEventually(t, ctx, clients.K8sClient),
webhookReadinessTimeout, webhookReadinessTick)
} else {
t.Log("not deploying operator to test cluster via kustomize")
}

t.Log("environment is ready, starting tests")

Expand Down Expand Up @@ -269,17 +291,26 @@ func deploymentAssertConditions(conds ...appsv1.DeploymentCondition) deploymentA
}
}

func waitForOperatorDeployment(ctx context.Context, k8sClient *kubernetes.Clientset, opts ...deploymentAssertOptions) error {
func waitForOperatorDeployment(ctx context.Context, ns string, k8sClient *kubernetes.Clientset, opts ...deploymentAssertOptions) error {
outer:
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
deployment, err := k8sClient.AppsV1().Deployments("kong-system").Get(ctx, "gateway-operator-controller-manager", metav1.GetOptions{})
listOpts := metav1.ListOptions{
LabelSelector: "app.kubernetes.io/name=gateway-operator",
}
deploymentList, err := k8sClient.AppsV1().Deployments(ns).List(ctx, listOpts)
if err != nil {
return err
}
if len(deploymentList.Items) == 0 {
continue
}

deployment := &deploymentList.Items[0]

if deployment.Status.AvailableReplicas <= 0 {
continue
}
Expand Down
134 changes: 134 additions & 0 deletions test/e2e/test_helm_install_upgrade.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package e2e

import (
"context"
"fmt"
"strings"
"testing"

"github.com/gruntwork-io/terratest/modules/helm"
"github.com/gruntwork-io/terratest/modules/k8s"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"
)

func init() {
addTestsToTestSuite(TestUpgrade)
}

func TestUpgrade(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

// createEnvironment will queue up environment cleanup if necessary
// and dumping diagnostics if the test fails.
e := CreateEnvironment(t, ctx)

const (
// https://github.com/Kong/charts/tree/main/charts/gateway-operator
chart = "kong/gateway-operator"
namespace = "kong"
)

testcases := []struct {
name string
from string
to string
upgradeToCurrent bool
baseValues map[string]string
}{
{
name: "upgrade from 1.1.0 to 1.2.1",
from: "1.1.0",
to: "1.2.1",
// Disable ControlPlane and Gateway controllers to avoid ControlPlane
// and GatewayConfiguration CRD upgrade issues.
// 1.2.0 introduced a breaking change (moving both of those CRDs
// from v1alpha1 to v1beta1) that would cause the upgrade to fail.
baseValues: map[string]string{
"args": "{--enable-controller-gateway=false,--enable-controller-controlplane=false}",
},
},
{
name: "upgrade from 1.2.1 to latest",
from: "1.2.1",
upgradeToCurrent: true,
},
}

for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
var imageTo string
if tc.upgradeToCurrent {
if imageLoad == "" && imageOverride == "" {
t.Skipf("No KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE nor KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD" +
" env specified. Please specify the image to upgrade to in order to run this test.")
}
if imageLoad != "" {
t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD set to %q, using it to upgrade", imageLoad)
imageTo = imageLoad
} else {
t.Logf("KONG_TEST_GATEWAY_OPERATOR_IMAGE_OVERRIDE set to %q, using it to upgrade", imageOverride)
imageTo = imageOverride
}
}

releaseName := strings.ReplaceAll(fmt.Sprintf("kgo-%s-to-%s", tc.from, tc.to), ".", "-")
values := make(map[string]string)
if tc.baseValues != nil {
values = tc.baseValues
}
values["image.tag"] = tc.from

opts := &helm.Options{
KubectlOptions: &k8s.KubectlOptions{
Namespace: e.Namespace.Name,
RestConfig: e.Environment.Cluster().Config(),
},
SetValues: values,
}

require.NoError(t, helm.AddRepoE(t, opts, "kong", "https://charts.konghq.com"))
require.NoError(t, helm.InstallE(t, opts, chart, releaseName))
t.Cleanup(func() {
out, err := helm.RunHelmCommandAndGetOutputE(t, opts, "uninstall", releaseName)
if !assert.NoError(t, err) {
t.Logf("output: %s", out)
}
})

require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient,
deploymentAssertConditions(deploymentReadyConditions()...),
))

opts.SetValues["image.tag"] = tc.to
if tc.upgradeToCurrent {
opts.SetValues["image.tag"] = imageTo
}

require.NoError(t, helm.UpgradeE(t, opts, chart, releaseName))
require.NoError(t, waitForOperatorDeployment(ctx, e.Namespace.Name, e.Clients.K8sClient,
deploymentAssertConditions(deploymentReadyConditions()...),
))

// TODO: Add preconditions checks
// time.Sleep(30 * time.Second)
})
}
}

func deploymentReadyConditions() []appsv1.DeploymentCondition {
return []appsv1.DeploymentCondition{
{
Reason: "NewReplicaSetAvailable",
Status: "True",
Type: "Progressing",
},
{
Reason: "MinimumReplicasAvailable",
Status: "True",
Type: "Available",
},
}
}
2 changes: 1 addition & 1 deletion test/e2e/test_operator_logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func TestOperatorLogs(t *testing.T) {

// createEnvironment will queue up environment cleanup if necessary
// and dumping diagnostics if the test fails.
e := CreateEnvironment(t, ctx)
e := CreateEnvironment(t, ctx, WithInstallViaKustomize())
clients, testNamespace, cleaner := e.Clients, e.Namespace, e.Cleaner

t.Log("finding the Pod for the Gateway Operator")
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/test_upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,13 +76,13 @@ func testManifestsUpgrade(
ctx context.Context,
testParams upgradeTestParams,
) {
e := CreateEnvironment(t, ctx, WithOperatorImage(testParams.fromImage))
e := CreateEnvironment(t, ctx, WithOperatorImage(testParams.fromImage), WithInstallViaKustomize())

kustomizationDir := PrepareKustomizeDir(t, testParams.toImage)
t.Logf("deploying operator %q to test cluster %q via kustomize", testParams.toImage, e.Environment.Name())
require.NoError(t, clusters.KustomizeDeployForCluster(ctx, e.Environment.Cluster(), kustomizationDir.Tests(), "--server-side", "-v5"))
t.Log("waiting for operator deployment to complete")
require.NoError(t, waitForOperatorDeployment(ctx, e.Clients.K8sClient,
require.NoError(t, waitForOperatorDeployment(ctx, "kong-system", e.Clients.K8sClient,
deploymentAssertConditions(
appsv1.DeploymentCondition{
Reason: "NewReplicaSetAvailable",
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/test_webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func TestDataPlaneValidatingWebhook(t *testing.T) {

// createEnvironment will queue up environment cleanup if necessary
// and dumping diagnostics if the test fails.
e := CreateEnvironment(t, ctx)
e := CreateEnvironment(t, ctx, WithInstallViaKustomize())
clients, testNamespace := e.Clients, e.Namespace

testCases := []struct {
Expand Down
8 changes: 7 additions & 1 deletion test/helpers/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,14 +75,20 @@ func ParseGoTestFlags(testRunner func(t *testing.T), testSuiteToRun []func(t *te
os.Exit(1)
}

fmt.Printf("test.run: %s\n", fRun.Value.String())

// Hack - set test.run flag to the name of the test function that runs the test suite
// to execute it with tests that are returned from this function.
// They are explicitly passed to RunTestSuite(...) as an argument.
if err := fRun.Value.Set(getFunctionName(testRunner)); err != nil {
if err := fRun.Value.Set(getFunctionName(testRunner) + "/" + fRun.Value.String()); err != nil {
fmt.Println("testing: unexpected error happened (it should never happen, check the code)", err)
os.Exit(1)
}

fmt.Printf("test.run: %s\n", fRun.Value.String())

// os.Exit(0)

return testSuiteToRun
}

Expand Down

0 comments on commit 7972254

Please sign in to comment.