feat: Adds a new quarantine package that has a function to quarantine…

… tests. Updates existing test skips to this quarantine test function. Closes [rancher/qa-tasks/issues/1484].
caliskanugur · Sep 26, 2024 · 2fc86c4 · 2fc86c4
1 parent 323c427
commit 2fc86c4
Show file tree

Hide file tree

Showing 10 changed files with 51 additions and 9 deletions.
diff --git a/go.mod b/go.mod
@@ -434,3 +434,5 @@ require (
 	sigs.k8s.io/kustomize/kyaml v0.17.1 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )
+
+replace github.com/rancher/shepherd => github.com/caliskanugur/shepherd v0.0.0-20240924173307-38b1457cbacb
diff --git a/go.sum b/go.sum
@@ -777,6 +777,8 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0Bsq
 github.com/bugsnag/panicwrap v0.0.0-20160118154447-aceac81c6e2f h1:7+HZb3PwI0cCUETyMDgPCvSta1y3idwBL8PHwSgelJk=
 github.com/bugsnag/panicwrap v0.0.0-20160118154447-aceac81c6e2f/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/caliskanugur/shepherd v0.0.0-20240924173307-38b1457cbacb h1:rfHGMxxm/2x5g2TffiU7uHqQZ+lyuJ7KBEYw5Sec3r0=
+github.com/caliskanugur/shepherd v0.0.0-20240924173307-38b1457cbacb/go.mod h1:zekxs8n6YwnsX1i58abObrkWDfdA9O3hV8wQATuS+8o=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -1521,8 +1523,6 @@ github.com/rancher/rke v1.6.0 h1:fHdygmtPF1cWXuiYXfwgG4hKvt0n4l57SwCxquRJSfs=
 github.com/rancher/rke v1.6.0/go.mod h1:5xRbf3L8PxqJRhABjYRfaBqbpVqAnqyH3maUNQEuwvk=
 github.com/rancher/saml v0.4.14-rancher3 h1:2NN6cPqm9FJeiT25x8+gLHWGdulsEak33cHRkGaJ5v0=
 github.com/rancher/saml v0.4.14-rancher3/go.mod h1:S4+611dxnKt8z/ulbvaJzcgSHsuhjVc1QHNTcr1R7Fw=
-github.com/rancher/shepherd v0.0.0-20240912175816-661a2b75cab3 h1:P95YlwQi8QUt9m6IHxyTSMAmWyW5+Uj6hBJZUdFYusc=
-github.com/rancher/shepherd v0.0.0-20240912175816-661a2b75cab3/go.mod h1:nVphr8v6qtXd0pth8wMCF9U5eKEPBIaD5+HQCH19uRw=
 github.com/rancher/steve v0.0.0-20240906202532-e930ae3691b2 h1:ZuEwMQ1sXJnrNP5a54huYPQJpaghcWiCxyD6WigUFY8=
 github.com/rancher/steve v0.0.0-20240906202532-e930ae3691b2/go.mod h1:U6MM2uv9zwOYp5JMNI/0GqHIiDd7H+9layTM5KBtpX4=
 github.com/rancher/system-upgrade-controller/pkg/apis v0.0.0-20240301001845-4eacc2dabbde h1:x5VZI/0TUx1MeZirh6e0OMAInhCmq6yRvD6897458Ng=

diff --git a/tests/v2/actions/quarantine/quarantine.go b/tests/v2/actions/quarantine/quarantine.go
@@ -0,0 +1,32 @@
+package quarantine
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/rancher/shepherd/pkg/environmentflag"
+)
+
+// Quarantine package's Test method skips the given testing T by default when it's used
+// to run quarantine flag, the flag in the configuration needs to be set like:
+//
+// flags:
+//   desiredflags: quarantined
+//
+// or multiple flags with quarantine option:
+//
+// flags:
+//   desiredflags: quarantined|long|otherFlag
+
+func Test(t *testing.T, args ...any) {
+	environmentFlags := environmentflag.NewEnvironmentFlags()
+	environmentflag.LoadEnvironmentFlags(environmentflag.ConfigurationFileKey, environmentFlags)
+
+	quarantined := fmt.Sprintf("Test [%v] is quarantined, skipping:", t.Name())
+
+	reason := append([]any{quarantined}, args...)
+
+	if !environmentFlags.GetValue(environmentflag.Quarantined) {
+		t.Skip(reason)
+	}
+}
diff --git a/tests/v2/integration/steveapi/steve_api_test.go b/tests/v2/integration/steveapi/steve_api_test.go
@@ -19,6 +19,7 @@ import (
 	"github.com/rancher/rancher/tests/v2/actions/kubeapi/rbac"
 	"github.com/rancher/rancher/tests/v2/actions/kubeapi/secrets"
 	"github.com/rancher/rancher/tests/v2/actions/namespaces"
+	"github.com/rancher/rancher/tests/v2/actions/quarantine"
 	stevesecrets "github.com/rancher/rancher/tests/v2/actions/secrets"
 	"github.com/rancher/rancher/tests/v2/actions/serviceaccounts"
 	"github.com/rancher/shepherd/clients/rancher"
@@ -2632,6 +2633,7 @@ func (s *steveAPITestSuite) TestCRUD() {
 		assert.Nil(s.T(), readObj)
 	})
 }
+
 func (s *steveAPITestSuite) assertListIsEqual(expect []map[string]string, list []clientv1.SteveAPIObject) {
 	assert.Equal(s.T(), len(expect), len(list))
 	for i, w := range expect {
@@ -2689,6 +2691,6 @@ func TestSteveLocal(t *testing.T) {
 
 func TestSteveDownstream(t *testing.T) {
 	// TODO: Re-enable the test when the bug is fixed
-	t.Skip()
+	quarantine.Test(t, "Re-enable the test when the bug is fixed, r/r#46391")
 	suite.Run(t, new(DownstreamSteveAPITestSuite))
 }
diff --git a/tests/v2/validation/nodescaling/scaling_node_driver_aks_test.go b/tests/v2/validation/nodescaling/scaling_node_driver_aks_test.go
@@ -5,6 +5,7 @@ package nodescaling
 import (
 	"testing"
 
+	"github.com/rancher/rancher/tests/v2/actions/quarantine"
 	"github.com/rancher/rancher/tests/v2/actions/scalinginput"
 	"github.com/rancher/shepherd/clients/rancher"
 	"github.com/rancher/shepherd/extensions/clusters"
@@ -81,6 +82,6 @@ func (s *AKSNodeScalingTestSuite) TestScalingAKSNodePoolsDynamicInput() {
 // In order for 'go test' to run this suite, we need to create
 // a normal test function and pass our suite to suite.Run
 func TestAKSNodeScalingTestSuite(t *testing.T) {
-	t.Skip("This test has been deprecated; check https://github.com/rancher/hosted-providers-e2e for updated tests")
+	quarantine.Test(t, "This test has been deprecated; check https://github.com/rancher/hosted-providers-e2e for updated tests")
 	suite.Run(t, new(AKSNodeScalingTestSuite))
 }
diff --git a/tests/v2/validation/nodescaling/scaling_node_driver_eks_test.go b/tests/v2/validation/nodescaling/scaling_node_driver_eks_test.go
@@ -5,6 +5,7 @@ package nodescaling
 import (
 	"testing"
 
+	"github.com/rancher/rancher/tests/v2/actions/quarantine"
 	"github.com/rancher/rancher/tests/v2/actions/scalinginput"
 	"github.com/rancher/shepherd/clients/rancher"
 	"github.com/rancher/shepherd/extensions/clusters"
@@ -81,6 +82,6 @@ func (s *EKSNodeScalingTestSuite) TestScalingEKSNodePoolsDynamicInput() {
 // In order for 'go test' to run this suite, we need to create
 // a normal test function and pass our suite to suite.Run
 func TestEKSNodeScalingTestSuite(t *testing.T) {
-	t.Skip("This test has been deprecated; check https://github.com/rancher/hosted-providers-e2e for updated tests")
+	quarantine.Test(t, "This test has been deprecated; check https://github.com/rancher/hosted-providers-e2e for updated tests")
 	suite.Run(t, new(EKSNodeScalingTestSuite))
 }
diff --git a/tests/v2/validation/nodescaling/scaling_node_driver_gke_test.go b/tests/v2/validation/nodescaling/scaling_node_driver_gke_test.go
@@ -5,6 +5,7 @@ package nodescaling
 import (
 	"testing"
 
+	"github.com/rancher/rancher/tests/v2/actions/quarantine"
 	"github.com/rancher/rancher/tests/v2/actions/scalinginput"
 	"github.com/rancher/shepherd/clients/rancher"
 	"github.com/rancher/shepherd/extensions/clusters"
@@ -81,6 +82,6 @@ func (s *GKENodeScalingTestSuite) TestScalingGKENodePoolsDynamicInput() {
 // In order for 'go test' to run this suite, we need to create
 // a normal test function and pass our suite to suite.Run
 func TestGKENodeScalingTestSuite(t *testing.T) {
-	t.Skip("This test has been deprecated; check https://github.com/rancher/hosted-providers-e2e for updated tests")
+	quarantine.Test(t, "This test has been deprecated; check https://github.com/rancher/hosted-providers-e2e for updated tests")
 	suite.Run(t, new(GKENodeScalingTestSuite))
 }
diff --git a/tests/v2/validation/provisioning/hosted/aks/hosted_provisioning_test.go b/tests/v2/validation/provisioning/hosted/aks/hosted_provisioning_test.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/rancher/rancher/tests/v2/actions/provisioning"
 	"github.com/rancher/rancher/tests/v2/actions/provisioninginput"
+	"github.com/rancher/rancher/tests/v2/actions/quarantine"
 	"github.com/rancher/rancher/tests/v2/actions/reports"
 	"github.com/rancher/shepherd/clients/rancher"
 	management "github.com/rancher/shepherd/clients/rancher/generated/management/v3"
@@ -86,6 +87,6 @@ func (h *HostedAKSClusterProvisioningTestSuite) TestProvisioningHostedAKS() {
 // In order for 'go test' to run this suite, we need to create
 // a normal test function and pass our suite to suite.Run
 func TestHostedAKSClusterProvisioningTestSuite(t *testing.T) {
-	t.Skip("This test has been deprecated; check https://github.com/rancher/hosted-providers-e2e for updated tests")
+	quarantine.Test(t, "This test has been deprecated; check https://github.com/rancher/hosted-providers-e2e for updated tests")
 	suite.Run(t, new(HostedAKSClusterProvisioningTestSuite))
 }
diff --git a/tests/v2/validation/provisioning/hosted/eks/hosted_provisioning_test.go b/tests/v2/validation/provisioning/hosted/eks/hosted_provisioning_test.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/rancher/rancher/tests/v2/actions/provisioning"
 	"github.com/rancher/rancher/tests/v2/actions/provisioninginput"
+	"github.com/rancher/rancher/tests/v2/actions/quarantine"
 	"github.com/rancher/rancher/tests/v2/actions/reports"
 	"github.com/rancher/shepherd/clients/rancher"
 	management "github.com/rancher/shepherd/clients/rancher/generated/management/v3"
@@ -85,6 +86,6 @@ func (h *HostedEKSClusterProvisioningTestSuite) TestProvisioningHostedEKS() {
 // In order for 'go test' to run this suite, we need to create
 // a normal test function and pass our suite to suite.Run
 func TestHostedEKSClusterProvisioningTestSuite(t *testing.T) {
-	t.Skip("This test has been deprecated; check https://github.com/rancher/hosted-providers-e2e for updated tests")
+	quarantine.Test(t, "This test has been deprecated; check https://github.com/rancher/hosted-providers-e2e for updated tests")
 	suite.Run(t, new(HostedEKSClusterProvisioningTestSuite))
 }
diff --git a/tests/v2/validation/provisioning/hosted/gke/hosted_provisioning_test.go b/tests/v2/validation/provisioning/hosted/gke/hosted_provisioning_test.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/rancher/rancher/tests/v2/actions/provisioning"
 	"github.com/rancher/rancher/tests/v2/actions/provisioninginput"
+	"github.com/rancher/rancher/tests/v2/actions/quarantine"
 	"github.com/rancher/rancher/tests/v2/actions/reports"
 	"github.com/rancher/shepherd/clients/rancher"
 	management "github.com/rancher/shepherd/clients/rancher/generated/management/v3"
@@ -84,6 +85,6 @@ func (h *HostedGKEClusterProvisioningTestSuite) TestProvisioningHostedGKE() {
 // In order for 'go test' to run this suite, we need to create
 // a normal test function and pass our suite to suite.Run
 func TestHostedGKEClusterProvisioningTestSuite(t *testing.T) {
-	t.Skip("This test has been deprecated; check https://github.com/rancher/hosted-providers-e2e for updated tests")
+	quarantine.Test(t, "This test has been deprecated; check https://github.com/rancher/hosted-providers-e2e for updated tests")
 	suite.Run(t, new(HostedGKEClusterProvisioningTestSuite))
 }