diff --git a/docs/running.md b/docs/running.md index 9267fd7..1e85c04 100644 --- a/docs/running.md +++ b/docs/running.md @@ -16,6 +16,7 @@ Options besides `-url`: * `-junit ` - writes test results in JUnit XML format to the specified file * `-debug` - enables verbose logging of test actions for failed tests * `-debug-all` - enables verbose logging of test actions for all tests +* `-enable-persistence-tests` - enables tests that require external persistence (e.g. a database like redis) * `-record-failures` - record failed test IDs to the given file. Recorded tests can be skipped by the next run of the harness via `-skip-from`. * `-skip-from` - skips any test IDs recorded in the specified file. May be used in conjunction with `-record-failures` diff --git a/docs/service_spec.md b/docs/service_spec.md index d6c174c..9ecd165 100644 --- a/docs/service_spec.md +++ b/docs/service_spec.md @@ -110,6 +110,16 @@ This means the SDK is capable of interacting with external persistent data store - `dynamodb` - `redis` +The persistence store tests are unique in that they rely on external services to be available prior to running the test harness. Because of this, the test harness will not run these tests by default, even if the capabilities exist, without also provided the `-enable-persistence-tests` flag. + +These services can be easily provided through the use of docker. Example commands are shown below for your convenience. + +```shell +docker run --network=host amazon/dynamodb-local +docker run --network=host hashicorp/consul +docker run --network host redis +``` + #### Capability `"polling-gzip"` This means the SDK is requesting gzip compression support on polling payloads. The SDK is expected to set the `Accept-Encoding` header to `gzip` in addition to enabling this capability. diff --git a/framework/harness/harness.go b/framework/harness/harness.go index 0f64c71..a55dcd9 100644 --- a/framework/harness/harness.go +++ b/framework/harness/harness.go @@ -108,6 +108,7 @@ func NewTestHarness( testServiceBaseURL string, testHarnessExternalHostname string, testHarnessPort int, + testHarnessEnablePersistenceTests bool, statusQueryTimeout time.Duration, debugLogger framework.Logger, startupOutput io.Writer, @@ -129,6 +130,21 @@ func NewTestHarness( if err != nil { return nil, err } + + // If we aren't running persistence tests, remove the capabilities that would enable it. + if !testHarnessEnablePersistenceTests { + filteredCapabilities := make([]string, 0, len(testServiceInfo.Capabilities)) + for _, c := range testServiceInfo.Capabilities { + if c == servicedef.CapabilityPersistentDataStoreRedis || + c == servicedef.CapabilityPersistentDataStoreDynamoDB || + c == servicedef.CapabilityPersistentDataStoreConsul { + debugLogger.Printf("Disabling capability %q because persistence tests are disabled", c) + continue + } + filteredCapabilities = append(filteredCapabilities, c) + } + testServiceInfo.Capabilities = filteredCapabilities + } h.testServiceInfo = testServiceInfo if err := startServer(testHarnessPort, http.HandlerFunc(h.serveHTTP)); err != nil { diff --git a/main.go b/main.go index 3aeccd7..94e9c61 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,7 @@ func run(params commandParams) (*ldtest.Results, error) { params.serviceURL, params.host, params.port, + params.enablePersistenceTests, time.Duration(params.queryTimeoutSeconds)*time.Second, mainDebugLogger, os.Stdout, diff --git a/params.go b/params.go index 081f027..231f898 100644 --- a/params.go +++ b/params.go @@ -9,17 +9,18 @@ import ( ) type commandParams struct { - serviceURL string - port int - host string - filters ldtest.RegexFilters - stopServiceAtEnd bool - debug bool - debugAll bool - jUnitFile string - recordFailures string - skipFile string - queryTimeoutSeconds int + serviceURL string + port int + host string + filters ldtest.RegexFilters + stopServiceAtEnd bool + debug bool + debugAll bool + enablePersistenceTests bool + jUnitFile string + recordFailures string + skipFile string + queryTimeoutSeconds int } func (c *commandParams) Read(args []string) bool { @@ -33,6 +34,7 @@ func (c *commandParams) Read(args []string) bool { fs.BoolVar(&c.stopServiceAtEnd, "stop-service-at-end", false, "tell test service to exit after the test run") fs.BoolVar(&c.debug, "debug", false, "enable debug logging for failed tests") fs.BoolVar(&c.debugAll, "debug-all", false, "enable debug logging for all tests") + fs.BoolVar(&c.enablePersistenceTests, "enable-persistence-tests", false, "enable tests that require external persistence support") fs.StringVar(&c.jUnitFile, "junit", "", "write JUnit XML output to the specified path") fs.StringVar(&c.recordFailures, "record-failures", "", "record failed test IDs to the given file.\n"+ "recorded tests can be skipped by the next run of the harness via -skip-from") diff --git a/sdktests/server_side_persistence_base.go b/sdktests/server_side_persistence_base.go index d7914fc..10017c6 100644 --- a/sdktests/server_side_persistence_base.go +++ b/sdktests/server_side_persistence_base.go @@ -41,7 +41,10 @@ const ( ) func doServerSidePersistentTests(t *ldtest.T) { + ranAtLeastOnce := false + if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreRedis) { + ranAtLeastOnce = true rdb := redis.NewClient(&redis.Options{ Addr: "localhost:6379", Password: "", // no password set @@ -52,6 +55,7 @@ func doServerSidePersistentTests(t *ldtest.T) { } if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreConsul) { + ranAtLeastOnce = true config := consul.DefaultConfig() config.Address = "localhost:8500" @@ -62,6 +66,7 @@ func doServerSidePersistentTests(t *ldtest.T) { } if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreDynamoDB) { + ranAtLeastOnce = true mySession := session.Must(session.NewSession( aws.NewConfig(). WithRegion("us-east-1"). @@ -81,6 +86,10 @@ func doServerSidePersistentTests(t *ldtest.T) { t.Run("dynamodb", newServerSidePersistentTests(t, &store, "").Run) } + + if !ranAtLeastOnce { + t.Skip() + } } type PersistentStore interface { diff --git a/sdktests/testsuite_entry_point.go b/sdktests/testsuite_entry_point.go index 4bb2f46..d07d9dd 100644 --- a/sdktests/testsuite_entry_point.go +++ b/sdktests/testsuite_entry_point.go @@ -92,12 +92,7 @@ func doAllServerSideTests(t *ldtest.T) { t.Run("migrations", doServerSideMigrationTests) t.Run("hooks", doCommonHooksTests) t.Run("wrapper", doServerSideWrapperTests) - - if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreRedis) || - t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreConsul) || - t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreDynamoDB) { - t.Run("persistent data store", doServerSidePersistentTests) - } + t.Run("persistent data store", doServerSidePersistentTests) } func doAllClientSideTests(t *ldtest.T) {