tests(conformance): run conformance against expressions and traditional_compatible router flavors

.github/workflows/tests.yaml

@@ -177,6 +177,12 @@ jobs:
 
   conformance-tests:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - router-flavor: traditional_compatible
+        - router-flavor: expressions
     steps:
     - name: checkout repository
       uses: actions/checkout@v4
@@ -196,7 +202,8 @@ jobs:
     - name: run conformance tests
       run: make test.conformance
       env:
-        GOTESTSUM_JUNITFILE: "conformance-tests.xml"
+        GOTESTSUM_JUNITFILE: conformance-tests-${{ matrix.router-flavor }}.xml
+        TEST_KONG_ROUTER_FLAVOR: ${{ matrix.router-flavor }}
 
     - name: upload diagnostics
       if: ${{ always() }}
@@ -211,7 +218,13 @@ jobs:
       uses: actions/upload-artifact@v3
       with:
         name: tests-report
-        path: conformance-tests.xml
+        path: conformance-tests-${{ matrix.router-flavor }}.xml
+
+    - name: collect conformance report
+      uses: actions/upload-artifact@v4
+      with:
+        name: conformance-report-${{ matrix.router-flavor }}
+        path: kong-gateway-operator*.yaml
 
   integration-tests:
     runs-on: ubuntu-latest

test/conformance/conformance_test.go

@@ -1,6 +1,8 @@
 package conformance
 
 import (
+	"context"
+	"fmt"
 	"testing"
 
 	"github.com/samber/lo"
@@ -16,14 +18,26 @@ import (
 	"sigs.k8s.io/gateway-api/conformance/utils/suite"
 	"sigs.k8s.io/gateway-api/pkg/features"
 
-	v1beta1 "github.com/kong/gateway-operator/api/v1beta1"
+	"github.com/kong/gateway-operator/api/v1beta1"
 	gwtypes "github.com/kong/gateway-operator/internal/types"
 	"github.com/kong/gateway-operator/modules/manager/metadata"
 	testutils "github.com/kong/gateway-operator/pkg/utils/test"
 	"github.com/kong/gateway-operator/pkg/vars"
 )
 
-var skippedTests = []string{
+var skippedTestsForExpressionsRouter = []string{
+	// gateway
+	tests.GatewayInvalidTLSConfiguration.ShortName,
+
+	// httproute
+	tests.HTTPRouteInvalidBackendRefUnknownKind.ShortName,
+
+	// TODO: remove the skip https://github.com/Kong/gateway-operator/issues/295
+	// This test is flaky.
+	tests.HTTPRouteWeight.ShortName,
+}
+
+var skippedTestsForTraditionalCompatibleRouter = []string{
 	// gateway
 	tests.GatewayInvalidTLSConfiguration.ShortName,
 
@@ -36,10 +50,73 @@ var skippedTests = []string{
 	tests.HTTPRouteWeight.ShortName,
 }
 
+type ConformanceConfig struct {
+	KongRouterFlavor RouterFlavor
+}
+
 func TestGatewayConformance(t *testing.T) {
 	t.Parallel()
 
-	t.Log("creating GatewayClass for gateway conformance tests")
+	// Conformance tests are run for both available router flavours:
+	// traditional_compatible and expressions.
+	var (
+		config       ConformanceConfig
+		skippedTests []string
+	)
+	switch rf := KongRouterFlavor(t); rf {
+	case RouterFlavorTraditionalCompatible:
+		skippedTests = skippedTestsForTraditionalCompatibleRouter
+		config.KongRouterFlavor = RouterFlavorTraditionalCompatible
+	case RouterFlavorExpressions:
+		skippedTests = skippedTestsForExpressionsRouter
+		config.KongRouterFlavor = RouterFlavorExpressions
+	default:
+		t.Fatalf("unsupported KongRouterFlavor: %s", rf)
+	}
+
+	t.Logf("using the following configuration for the conformance tests: %+v", config)
+
+	t.Log("creating GatewayConfiguration and GatewayClass for gateway conformance tests")
+	gwconf := createGatewayConfiguration(ctx, t, config)
+	gwc := createGatewayClass(ctx, t, gwconf)
+
+	// There are no explicit conformance tests for GatewayClass, but we can
+	// still run the conformance test suite setup to ensure that the
+	// GatewayClass gets accepted.
+	t.Log("configuring the Gateway API conformance test suite")
+	// Currently mode only relies on the KongRouterFlavor, but in the future
+	// we may want to add more modes.
+	mode := string(config.KongRouterFlavor)
+	reportFileName := fmt.Sprintf("kong-gateway-operator-%s.yaml", mode) // TODO: https://github.com/Kong/gateway-operator/issues/268
+
+	opts := conformance.DefaultOptions(t)
+	opts.ReportOutputPath = "../../" + reportFileName
+	opts.Client = clients.MgrClient
+	opts.GatewayClassName = gwc.Name
+	opts.BaseManifests = testutils.GatewayRawRepoURL + "/conformance/base/manifests.yaml"
+	opts.SkipTests = skippedTests
+	opts.Mode = mode
+	opts.ConformanceProfiles = sets.New(
+		suite.GatewayHTTPConformanceProfileName,
+	)
+	opts.SupportedFeatures = sets.New(
+		features.SupportHTTPRouteResponseHeaderModification,
+	)
+	opts.Implementation = conformancev1.Implementation{
+		Organization: metadata.Organization,
+		Project:      metadata.ProjectName,
+		URL:          metadata.RepoURL,
+		Version:      metadata.Release,
+		Contact: []string{
+			metadata.RepoURL + "/issues/new/choose",
+		},
+	}
+
+	t.Log("running the Gateway API conformance test suite")
+	conformance.RunConformanceWithOptions(t, opts)
+}
+
+func createGatewayConfiguration(ctx context.Context, t *testing.T, c ConformanceConfig) *v1beta1.GatewayConfiguration {
 	gwconf := v1beta1.GatewayConfiguration{
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: "kgo-gwconf-conformance-",
@@ -68,6 +145,12 @@ func TestGatewayConformance(t *testing.T) {
 												corev1.ResourceMemory: resource.MustParse("1024Mi"),
 											},
 										},
+										Env: []corev1.EnvVar{
+											{
+												Name:  "KONG_ROUTER_FLAVOR",
+												Value: string(c.KongRouterFlavor),
+											},
+										},
 									},
 								},
 							},
@@ -113,11 +196,15 @@ func TestGatewayConformance(t *testing.T) {
 			},
 		},
 	}
+
 	require.NoError(t, clients.MgrClient.Create(ctx, &gwconf))
 	t.Cleanup(func() {
 		require.NoError(t, clients.MgrClient.Delete(ctx, &gwconf))
 	})
+	return &gwconf
+}
 
+func createGatewayClass(ctx context.Context, t *testing.T, gwconf *v1beta1.GatewayConfiguration) *gatewayv1.GatewayClass {
 	gwc := &gatewayv1.GatewayClass{
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: "kgo-gwclass-conformance-",
@@ -128,7 +215,7 @@ func TestGatewayConformance(t *testing.T) {
 				Group:     "gateway-operator.konghq.com",
 				Kind:      "GatewayConfiguration",
 				Name:      gwconf.Name,
-				Namespace: lo.ToPtr(gwtypes.Namespace("default")),
+				Namespace: lo.ToPtr(gwtypes.Namespace(gwconf.Namespace)),
 			},
 		},
 	}
@@ -137,34 +224,5 @@ func TestGatewayConformance(t *testing.T) {
 		require.NoError(t, clients.MgrClient.Delete(ctx, gwc))
 	})
 
-	// There are no explicit conformance tests for GatewayClass, but we can
-	// still run the conformance test suite setup to ensure that the
-	// GatewayClass gets accepted.
-	t.Log("configuring the Gateway API conformance test suite")
-	const reportFileName = "kong-gateway-operator.yaml" // TODO: https://github.com/Kong/gateway-operator/issues/268
-
-	opts := conformance.DefaultOptions(t)
-	opts.ReportOutputPath = "../../" + reportFileName
-	opts.Client = clients.MgrClient
-	opts.GatewayClassName = gwc.Name
-	opts.BaseManifests = testutils.GatewayRawRepoURL + "/conformance/base/manifests.yaml"
-	opts.SkipTests = skippedTests
-	opts.ConformanceProfiles = sets.New(
-		suite.GatewayHTTPConformanceProfileName,
-	)
-	opts.SupportedFeatures = sets.New(
-		features.SupportHTTPRouteResponseHeaderModification,
-	)
-	opts.Implementation = conformancev1.Implementation{
-		Organization: metadata.Organization,
-		Project:      metadata.ProjectName,
-		URL:          metadata.RepoURL,
-		Version:      metadata.Release,
-		Contact: []string{
-			metadata.RepoURL + "/issues/new/choose",
-		},
-	}
-
-	t.Log("running the Gateway API conformance test suite")
-	conformance.RunConformanceWithOptions(t, opts)
+	return gwc
 }

test/conformance/router_flavor.go

@@ -0,0 +1,41 @@
+package conformance
+
+import (
+	"os"
+	"testing"
+)
+
+// RouterFlavor represents the flavor of the Kong router.
+// ref: https://docs.konghq.com/gateway/latest/reference/configuration/#router_flavor
+type RouterFlavor string
+
+const (
+	// RouterFlavorTraditionalCompatible is the traditional compatible router flavor.
+	RouterFlavorTraditionalCompatible RouterFlavor = "traditional_compatible"
+	// RouterFlavorExpressions is the expressions router flavor.
+	RouterFlavorExpressions RouterFlavor = "expressions"
+)
+
+// KongRouterFlavor returns router mode of Kong in tests. Currently supports:
+// - `traditional`
+// - `traditional_compatible`.
+// - `expressions` (experimental, only for testing expression route related tests).
+func KongRouterFlavor(t *testing.T) RouterFlavor {
+	rf := os.Getenv("TEST_KONG_ROUTER_FLAVOR")
+	switch {
+	case rf == "":
+		return RouterFlavorTraditionalCompatible
+	case rf == string(RouterFlavorTraditionalCompatible):
+		return RouterFlavorTraditionalCompatible
+	case rf == string(RouterFlavorExpressions):
+		return RouterFlavorExpressions
+	case rf == "traditional":
+		t.Logf("Kong router flavor 'traditional' is deprecated, please use 'traditional_compatible' instead")
+		t.FailNow()
+		return ""
+	default:
+		t.Errorf("unsupported Kong router flavor: %s", rf)
+		t.FailNow()
+		return ""
+	}
+}