From bc4251e8b3eb8f495fb93ea7855445f4a576872d Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 3 Jun 2024 22:12:38 +0000 Subject: [PATCH 01/10] Initial end to end scenario test Signed-off-by: Peter Nied --- .../framework/SimpleRestoreFromSnapshot.java | 38 +++++++ .../SimpleRestoreFromSnapshot_ES_7_10.java | 4 +- .../com/rfs/integration/EndToEndTest.java | 105 ++++++++++++++++++ 3 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot.java create mode 100644 RFS/src/test/java/com/rfs/integration/EndToEndTest.java diff --git a/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot.java b/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot.java new file mode 100644 index 000000000..4b6249efb --- /dev/null +++ b/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot.java @@ -0,0 +1,38 @@ +package com.rfs.framework; + +import java.nio.file.Path; +import java.util.List; + +import com.rfs.common.ConnectionDetails; +import com.rfs.common.IndexMetadata; +import com.rfs.common.OpenSearchClient; + +public interface SimpleRestoreFromSnapshot { + + public static SimpleRestoreFromSnapshot forCluster(final String sourceClusterUrl) { + // TODO: determine version from source cluster + return new SimpleRestoreFromSnapshot_ES_7_10(); + } + + public default void fullMigrationViaLocalSnapshot(final String snapshotName, final String targetClusterUrl) throws Exception { + // TODO: Dynamically create / clean these up during tests + final var compressedSnapshotDirectory = ""; + final var unpackedShardDataDir = Path.of(""); + final var indices = extractSnapshotIndexData(compressedSnapshotDirectory, snapshotName, unpackedShardDataDir); + + final var targetClusterClient = new OpenSearchClient(new ConnectionDetails(targetClusterUrl, null, null)); + + // TODO: This should update the following metdata: + // - Global cluster state + // - Index Templates + // - Indices + // - Documents + + updateTargetCluster(indices, unpackedShardDataDir, targetClusterClient); + } + + public List extractSnapshotIndexData(final String localPath, final String snapshotName, final Path unpackedShardDataDir) throws Exception; + + public void updateTargetCluster(final List indices, final Path unpackedShardDataDir, final OpenSearchClient client) throws Exception; + +} diff --git a/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot_ES_7_10.java b/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot_ES_7_10.java index 82b6dbecc..d26346a3c 100644 --- a/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot_ES_7_10.java +++ b/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot_ES_7_10.java @@ -21,11 +21,11 @@ /** * Simplified version of RFS for use in testing - ES 7.10 version. */ -public class SimpleRestoreFromSnapshot_ES_7_10 { +public class SimpleRestoreFromSnapshot_ES_7_10 implements SimpleRestoreFromSnapshot { private static final Logger logger = LogManager.getLogger(SimpleRestoreFromSnapshot_ES_7_10.class); - public List extraSnapshotIndexData(final String localPath, final String snapshotName, final Path unpackedShardDataDir) throws Exception { + public List extractSnapshotIndexData(final String localPath, final String snapshotName, final Path unpackedShardDataDir) throws Exception { IOUtils.rm(unpackedShardDataDir); final var repo = new FileSystemRepo(Path.of(localPath)); diff --git a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java new file mode 100644 index 000000000..f3b7af4d4 --- /dev/null +++ b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java @@ -0,0 +1,105 @@ +package com.rfs.integration; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.mockito.ArgumentCaptor; + +import com.rfs.common.OpenSearchClient; +import com.rfs.framework.ClusterOperations; +import com.rfs.framework.ElasticsearchContainer; +import com.rfs.framework.SimpleRestoreFromSnapshot; +import com.rfs.framework.SimpleRestoreFromSnapshot_ES_6_8; +import com.rfs.framework.SimpleRestoreFromSnapshot_ES_7_10; +import com.rfs.framework.ElasticsearchContainer.Version; + +import lombok.Builder; +import lombok.Data; +import reactor.core.publisher.Mono; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.anyOf; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; + +import java.io.File; +import java.util.List; +import java.nio.file.Path; +import java.util.stream.Stream; +import java.util.function.Supplier; + +/** + * Tests focused on setting up whole source clusters, performing a migration, and validation on the target cluster + */ +public class EndToEndTest { + + public Object sourceCluster; + public Object targetCluster; + public SimpleRestoreFromSnapshot simpleRfsInstance; + + @Test + public void ES_v6_8_to_OS_v2_X_Migration() throws Exception { + // Setup + // PSUEDO: Create a source cluster running ES 6.8 + // PSUEOD: Update global cluster state + // - Change a 6.8 relevant setting + // PSUEOD: Create 2 index templates on the cluster + // - logs-* + // - data-rolling + // PSUEOD: Create 5 indices on the cluster + // - logs-01-2345 + // - logs-12-3456 + // - data-rolling + // - playground + // - playground2 + // PSUEDO: Add documents + // - 19x http-data docs into logs-01-2345 + // - 23x http-data docs into logs-12-3456 + // - 29x data-rolling + // - 5x geonames docs into playground + // - 7x geopoint into playground2 + + // PSUEDO: Create a target cluster running OS 2.X (Where x is the latest released version) + + // Action + // PSUEDO: Create snapshot on source cluster + // PSUEDO: Migrate from the snapshot + // PSUEDO: Shutdown source cluster + + // Validation + + // PSUEOD: Verfiy global cluster state + // - 6.8 Setting is transformed + // PSUEOD: Verify creation of 2 index templates on the cluster + // PSUEOD: Verify creation of 5 indices on the cluster + // - logs-01-2345 + // - logs-12-3456 + // - data-rolling + // - playground + // - playground2 + // PSUEDO: Verify documents + + // PSUEDO: Additional validation: + // - Mapping type parameter is removed + // + + // Looking for inspriation? + // - OS Breaking Changes https://opensearch.org/docs/latest/breaking-changes/ + // - ES Breaking Changes https://www.elastic.co/guide/en/elasticsearch/reference/7.0/breaking-changes-7.0.html + } +} From c2497d0657cff2f51fd9d1a35c472c7facfd07c2 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 3 Jun 2024 22:16:10 +0000 Subject: [PATCH 02/10] Remove snapshot name from interfaces Signed-off-by: Peter Nied --- .../java/com/rfs/framework/SimpleRestoreFromSnapshot.java | 6 +++--- RFS/src/test/java/com/rfs/integration/EndToEndTest.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot.java b/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot.java index 4b6249efb..9fd74a0e6 100644 --- a/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot.java +++ b/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot.java @@ -14,12 +14,12 @@ public static SimpleRestoreFromSnapshot forCluster(final String sourceClusterUrl return new SimpleRestoreFromSnapshot_ES_7_10(); } - public default void fullMigrationViaLocalSnapshot(final String snapshotName, final String targetClusterUrl) throws Exception { + public default void fullMigrationViaLocalSnapshot(final String targetClusterUrl) throws Exception { // TODO: Dynamically create / clean these up during tests + final var tempSnapshotName = ""; final var compressedSnapshotDirectory = ""; final var unpackedShardDataDir = Path.of(""); - final var indices = extractSnapshotIndexData(compressedSnapshotDirectory, snapshotName, unpackedShardDataDir); - + final var indices = extractSnapshotIndexData(compressedSnapshotDirectory, tempSnapshotName, unpackedShardDataDir); final var targetClusterClient = new OpenSearchClient(new ConnectionDetails(targetClusterUrl, null, null)); // TODO: This should update the following metdata: diff --git a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java index f3b7af4d4..0863e69e0 100644 --- a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java +++ b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java @@ -77,8 +77,8 @@ public void ES_v6_8_to_OS_v2_X_Migration() throws Exception { // PSUEDO: Create a target cluster running OS 2.X (Where x is the latest released version) // Action - // PSUEDO: Create snapshot on source cluster // PSUEDO: Migrate from the snapshot + simpleRfsInstance.fullMigrationViaLocalSnapshot(targetCluster.toString()); // PSUEDO: Shutdown source cluster // Validation From 1e5e1bf9323c64a58e7ea11ebd82d929660de342 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 3 Jun 2024 22:18:35 +0000 Subject: [PATCH 03/10] Clean up imports Signed-off-by: Peter Nied --- .../com/rfs/integration/EndToEndTest.java | 40 ------------------- 1 file changed, 40 deletions(-) diff --git a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java index 0863e69e0..e1f32f626 100644 --- a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java +++ b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java @@ -1,47 +1,7 @@ package com.rfs.integration; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; -import org.mockito.ArgumentCaptor; - -import com.rfs.common.OpenSearchClient; -import com.rfs.framework.ClusterOperations; -import com.rfs.framework.ElasticsearchContainer; import com.rfs.framework.SimpleRestoreFromSnapshot; -import com.rfs.framework.SimpleRestoreFromSnapshot_ES_6_8; -import com.rfs.framework.SimpleRestoreFromSnapshot_ES_7_10; -import com.rfs.framework.ElasticsearchContainer.Version; - -import lombok.Builder; -import lombok.Data; -import reactor.core.publisher.Mono; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; -import static org.hamcrest.CoreMatchers.allOf; -import static org.hamcrest.CoreMatchers.anyOf; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.MatcherAssert.assertThat; - -import java.io.File; -import java.util.List; -import java.nio.file.Path; -import java.util.stream.Stream; -import java.util.function.Supplier; /** * Tests focused on setting up whole source clusters, performing a migration, and validation on the target cluster From 562c50b6f0f99da6e25b6503c926254359756d24 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 3 Jun 2024 22:19:45 +0000 Subject: [PATCH 04/10] Embaressing spelling mistake Signed-off-by: Peter Nied --- .../com/rfs/integration/EndToEndTest.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java index e1f32f626..61cdb4396 100644 --- a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java +++ b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java @@ -15,46 +15,46 @@ public class EndToEndTest { @Test public void ES_v6_8_to_OS_v2_X_Migration() throws Exception { // Setup - // PSUEDO: Create a source cluster running ES 6.8 - // PSUEOD: Update global cluster state + // PSEUDO: Create a source cluster running ES 6.8 + // PSEUDO: Update global cluster state // - Change a 6.8 relevant setting - // PSUEOD: Create 2 index templates on the cluster + // PSEUDO: Create 2 index templates on the cluster // - logs-* // - data-rolling - // PSUEOD: Create 5 indices on the cluster + // PSEUDO: Create 5 indices on the cluster // - logs-01-2345 // - logs-12-3456 // - data-rolling // - playground // - playground2 - // PSUEDO: Add documents + // PSEUDO: Add documents // - 19x http-data docs into logs-01-2345 // - 23x http-data docs into logs-12-3456 // - 29x data-rolling // - 5x geonames docs into playground // - 7x geopoint into playground2 - // PSUEDO: Create a target cluster running OS 2.X (Where x is the latest released version) + // PSEUDO: Create a target cluster running OS 2.X (Where x is the latest released version) // Action - // PSUEDO: Migrate from the snapshot + // PSEUDO: Migrate from the snapshot simpleRfsInstance.fullMigrationViaLocalSnapshot(targetCluster.toString()); - // PSUEDO: Shutdown source cluster + // PSEUDO: Shutdown source cluster // Validation - // PSUEOD: Verfiy global cluster state + // PSEUDO: Verfiy global cluster state // - 6.8 Setting is transformed - // PSUEOD: Verify creation of 2 index templates on the cluster - // PSUEOD: Verify creation of 5 indices on the cluster + // PSEUDO: Verify creation of 2 index templates on the cluster + // PSEUDO: Verify creation of 5 indices on the cluster // - logs-01-2345 // - logs-12-3456 // - data-rolling // - playground // - playground2 - // PSUEDO: Verify documents + // PSEUDO: Verify documents - // PSUEDO: Additional validation: + // PSEUDO: Additional validation: // - Mapping type parameter is removed // From cad7e545abd85d62a7bdd9b9eb89a43c88392410 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 5 Jun 2024 20:08:36 +0000 Subject: [PATCH 05/10] Fix build break Signed-off-by: Peter Nied --- .../test/java/com/rfs/integration/SnapshotStateTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java b/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java index 39c1f9d47..470de3c5c 100644 --- a/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java +++ b/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java @@ -72,7 +72,7 @@ public void SingleSnapshot_SingleDocument() throws Exception { cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data"); - final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); + final var indices = srfs.extractSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); final var client = mock(OpenSearchClient.class); when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty()); @@ -105,7 +105,7 @@ public void SingleSnapshot_SingleDocument_Then_DeletedDocument() throws Exceptio cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data"); - final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); + final var indices = srfs.extractSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); final var client = mock(OpenSearchClient.class); when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty()); @@ -140,7 +140,7 @@ public void SingleSnapshot_SingleDocument_Then_UpdateDocument() throws Exception cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data"); - final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); + final var indices = srfs.extractSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); final var client = mock(OpenSearchClient.class); when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty()); From b93a5f3917b1fd6763400a3281e5fb0f29556245 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 5 Jun 2024 20:14:42 +0000 Subject: [PATCH 06/10] SAdd list of supported target containers Signed-off-by: Peter Nied --- .../rfs/framework/OpenSearchContainer.java | 59 +++++++++++++++++++ .../com/rfs/integration/EndToEndTest.java | 36 ++++++----- .../integration/SupportedTargetCluster.java | 22 +++++++ 3 files changed, 103 insertions(+), 14 deletions(-) create mode 100644 RFS/src/test/java/com/rfs/framework/OpenSearchContainer.java create mode 100644 RFS/src/test/java/com/rfs/integration/SupportedTargetCluster.java diff --git a/RFS/src/test/java/com/rfs/framework/OpenSearchContainer.java b/RFS/src/test/java/com/rfs/framework/OpenSearchContainer.java new file mode 100644 index 000000000..7184785e6 --- /dev/null +++ b/RFS/src/test/java/com/rfs/framework/OpenSearchContainer.java @@ -0,0 +1,59 @@ +package com.rfs.framework; + +import java.time.Duration; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +import lombok.extern.slf4j.Slf4j; + +/** + * Containerized version of OpenSearch cluster + */ +@Slf4j +public class OpenSearchContainer implements AutoCloseable { + + private final GenericContainer container; + private final Version version; + + @SuppressWarnings("resource") + public OpenSearchContainer(final Version version) { + this.version = version; + container = new GenericContainer<>(DockerImageName.parse(this.version.imageName)) + .withExposedPorts(9200, 9300) + .withEnv("discovery.type", "single-node") + .withEnv("plugins.security.disabled", "true") + .waitingFor(Wait.forHttp("/").forPort(9200).forStatusCode(200).withStartupTimeout(Duration.ofMinutes(1))); + } + + public String getUrl() { + final var address = container.getHost(); + final var port = container.getMappedPort(9200); + return "http://" + address + ":" + port; + } + + @Override + public void close() throws Exception { + log.info("Stopping version:" + version.prettyName); + log.debug("Instance logs:\n" + container.getLogs()); + container.stop(); + } + + public static enum Version { + V1_3_15("opensearchproject/opensearch:1.3.16", "1.3.16"), + V2_14_0("opensearchproject/opensearch:2.14.0", "2.14.0"); + + final String imageName; + final String prettyName; + Version(final String imageName, final String prettyName) { + this.imageName = imageName; + this.prettyName = prettyName; + } + + @Override + public String toString() { + return prettyName; + } + } +} diff --git a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java index 61cdb4396..17c3596fe 100644 --- a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java +++ b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java @@ -1,6 +1,9 @@ package com.rfs.integration; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ArgumentsSource; + +import com.rfs.framework.OpenSearchContainer; import com.rfs.framework.SimpleRestoreFromSnapshot; /** @@ -8,16 +11,15 @@ */ public class EndToEndTest { - public Object sourceCluster; - public Object targetCluster; - public SimpleRestoreFromSnapshot simpleRfsInstance; + protected Object sourceCluster; + protected Object targetCluster; + protected SimpleRestoreFromSnapshot simpleRfsInstance; - @Test - public void ES_v6_8_to_OS_v2_X_Migration() throws Exception { + @ParameterizedTest(name = "Target OpenSearch {0}") + @ArgumentsSource(SupportedTargetCluster.class) + public void ES_v6_8_Migration(final OpenSearchContainer.Version targetVersion) throws Exception { // Setup // PSEUDO: Create a source cluster running ES 6.8 - // PSEUDO: Update global cluster state - // - Change a 6.8 relevant setting // PSEUDO: Create 2 index templates on the cluster // - logs-* // - data-rolling @@ -38,13 +40,11 @@ public void ES_v6_8_to_OS_v2_X_Migration() throws Exception { // Action // PSEUDO: Migrate from the snapshot - simpleRfsInstance.fullMigrationViaLocalSnapshot(targetCluster.toString()); + // simpleRfsInstance.fullMigrationViaLocalSnapshot(targetCluster.toString()); // PSEUDO: Shutdown source cluster // Validation - // PSEUDO: Verfiy global cluster state - // - 6.8 Setting is transformed // PSEUDO: Verify creation of 2 index templates on the cluster // PSEUDO: Verify creation of 5 indices on the cluster // - logs-01-2345 @@ -57,9 +57,17 @@ public void ES_v6_8_to_OS_v2_X_Migration() throws Exception { // PSEUDO: Additional validation: // - Mapping type parameter is removed // + } + + @ParameterizedTest(name = "Target OpenSearch {0}") + @ArgumentsSource(SupportedTargetCluster.class) + public void ES_v7_10_Migration(final OpenSearchContainer.Version targetVersion) throws Exception { + // Placeholder + } - // Looking for inspriation? - // - OS Breaking Changes https://opensearch.org/docs/latest/breaking-changes/ - // - ES Breaking Changes https://www.elastic.co/guide/en/elasticsearch/reference/7.0/breaking-changes-7.0.html + @ParameterizedTest(name = "Target OpenSearch {0}") + @ArgumentsSource(SupportedTargetCluster.class) + public void ES_v7_17_Migration(final OpenSearchContainer.Version targetVersion) throws Exception { + // Placeholder } } diff --git a/RFS/src/test/java/com/rfs/integration/SupportedTargetCluster.java b/RFS/src/test/java/com/rfs/integration/SupportedTargetCluster.java new file mode 100644 index 000000000..a8d9d511e --- /dev/null +++ b/RFS/src/test/java/com/rfs/integration/SupportedTargetCluster.java @@ -0,0 +1,22 @@ +package com.rfs.integration; + +import java.util.stream.Stream; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; + +import com.rfs.framework.OpenSearchContainer; + +/** + * Defines all supported target clusters + */ +public class SupportedTargetCluster implements ArgumentsProvider { + + @Override + public Stream provideArguments(final ExtensionContext context) { + return Stream.of( + Arguments.of(OpenSearchContainer.Version.V1_3_15), + Arguments.of(OpenSearchContainer.Version.V2_14_0) + ); + } +} \ No newline at end of file From 7bab53c11f588bf708002dbd18a872c91654f4af Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 5 Jun 2024 20:29:09 +0000 Subject: [PATCH 07/10] Functional OS cluster startup Signed-off-by: Peter Nied --- .../java/com/rfs/framework/OpenSearchContainer.java | 10 ++++++++-- .../test/java/com/rfs/integration/EndToEndTest.java | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/RFS/src/test/java/com/rfs/framework/OpenSearchContainer.java b/RFS/src/test/java/com/rfs/framework/OpenSearchContainer.java index 7184785e6..7a2241c1b 100644 --- a/RFS/src/test/java/com/rfs/framework/OpenSearchContainer.java +++ b/RFS/src/test/java/com/rfs/framework/OpenSearchContainer.java @@ -21,9 +21,10 @@ public class OpenSearchContainer implements AutoCloseable { public OpenSearchContainer(final Version version) { this.version = version; container = new GenericContainer<>(DockerImageName.parse(this.version.imageName)) - .withExposedPorts(9200, 9300) + .withExposedPorts(9200) .withEnv("discovery.type", "single-node") .withEnv("plugins.security.disabled", "true") + .withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "SecurityIsDisabled123$%^") .waitingFor(Wait.forHttp("/").forPort(9200).forStatusCode(200).withStartupTimeout(Duration.ofMinutes(1))); } @@ -33,9 +34,14 @@ public String getUrl() { return "http://" + address + ":" + port; } + public void start() { + log.info("Starting version: " + version); + container.start(); + } + @Override public void close() throws Exception { - log.info("Stopping version:" + version.prettyName); + log.info("Stopping version: " + version); log.debug("Instance logs:\n" + container.getLogs()); container.stop(); } diff --git a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java index 17c3596fe..ed375ef41 100644 --- a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java +++ b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java @@ -6,9 +6,12 @@ import com.rfs.framework.OpenSearchContainer; import com.rfs.framework.SimpleRestoreFromSnapshot; +import lombok.extern.slf4j.Slf4j; + /** * Tests focused on setting up whole source clusters, performing a migration, and validation on the target cluster */ +@Slf4j public class EndToEndTest { protected Object sourceCluster; From e2a40379222e3bf9c0baf1392b9d91f46995f38a Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 5 Jun 2024 20:30:30 +0000 Subject: [PATCH 08/10] Disable tests that are inactive Signed-off-by: Peter Nied --- RFS/src/test/java/com/rfs/integration/EndToEndTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java index ed375ef41..6e03fbcb9 100644 --- a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java +++ b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java @@ -1,5 +1,7 @@ package com.rfs.integration; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; @@ -20,6 +22,7 @@ public class EndToEndTest { @ParameterizedTest(name = "Target OpenSearch {0}") @ArgumentsSource(SupportedTargetCluster.class) + @Disabled public void ES_v6_8_Migration(final OpenSearchContainer.Version targetVersion) throws Exception { // Setup // PSEUDO: Create a source cluster running ES 6.8 @@ -64,12 +67,14 @@ public void ES_v6_8_Migration(final OpenSearchContainer.Version targetVersion) t @ParameterizedTest(name = "Target OpenSearch {0}") @ArgumentsSource(SupportedTargetCluster.class) + @Disabled public void ES_v7_10_Migration(final OpenSearchContainer.Version targetVersion) throws Exception { // Placeholder } @ParameterizedTest(name = "Target OpenSearch {0}") @ArgumentsSource(SupportedTargetCluster.class) + @Disabled public void ES_v7_17_Migration(final OpenSearchContainer.Version targetVersion) throws Exception { // Placeholder } From da71806a83ae31db2aeda31ae54a5b51bcf0a49a Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 5 Jun 2024 20:41:42 +0000 Subject: [PATCH 09/10] Use common 7.X test method Signed-off-by: Peter Nied --- .../com/rfs/integration/EndToEndTest.java | 67 ++++++++++++++++--- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java index 6e03fbcb9..8dc6d24f9 100644 --- a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java +++ b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java @@ -5,6 +5,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; +import com.rfs.framework.ElasticsearchContainer; import com.rfs.framework.OpenSearchContainer; import com.rfs.framework.SimpleRestoreFromSnapshot; @@ -26,7 +27,7 @@ public class EndToEndTest { public void ES_v6_8_Migration(final OpenSearchContainer.Version targetVersion) throws Exception { // Setup // PSEUDO: Create a source cluster running ES 6.8 - // PSEUDO: Create 2 index templates on the cluster + // PSEUDO: Create 2 templates on the cluster, see https://www.elastic.co/guide/en/elasticsearch/reference/6.8/indices-templates.html // - logs-* // - data-rolling // PSEUDO: Create 5 indices on the cluster @@ -35,8 +36,8 @@ public void ES_v6_8_Migration(final OpenSearchContainer.Version targetVersion) t // - data-rolling // - playground // - playground2 - // PSEUDO: Add documents - // - 19x http-data docs into logs-01-2345 + // PSEUDO: Add documents + // - 19x http-data docs into logs-01-2345 // - 23x http-data docs into logs-12-3456 // - 29x data-rolling // - 5x geonames docs into playground @@ -46,7 +47,7 @@ public void ES_v6_8_Migration(final OpenSearchContainer.Version targetVersion) t // Action // PSEUDO: Migrate from the snapshot - // simpleRfsInstance.fullMigrationViaLocalSnapshot(targetCluster.toString()); + // simpleRfsInstance.fullMigrationViaLocalSnapshot(targetCluster.toString()); // PSEUDO: Shutdown source cluster // Validation @@ -59,23 +60,69 @@ public void ES_v6_8_Migration(final OpenSearchContainer.Version targetVersion) t // - playground // - playground2 // PSEUDO: Verify documents - + // PSEUDO: Additional validation: // - Mapping type parameter is removed - // + // } @ParameterizedTest(name = "Target OpenSearch {0}") @ArgumentsSource(SupportedTargetCluster.class) @Disabled - public void ES_v7_10_Migration(final OpenSearchContainer.Version targetVersion) throws Exception { - // Placeholder + public void migrateFrom_ES_v7_10(final OpenSearchContainer.Version targetVersion) throws Exception { + // Setup + // PSEUDO: Create a source cluster running ES 6.8 + + migrateFrom_ES_v7_X(null); } @ParameterizedTest(name = "Target OpenSearch {0}") @ArgumentsSource(SupportedTargetCluster.class) @Disabled - public void ES_v7_17_Migration(final OpenSearchContainer.Version targetVersion) throws Exception { - // Placeholder + public void migrateFrom_ES_v7_17(final OpenSearchContainer.Version targetVersion) throws Exception { + // Setup + // PSEUDO: Create a source cluster running ES 6.8 + + migrateFrom_ES_v7_X(null); + } + + private void migrateFrom_ES_v7_X(final ElasticsearchContainer sourceCluster) { + // PSEUDO: Create 2 index templates on the cluster, see https://www.elastic.co/guide/en/elasticsearch/reference/7.17/index-templates.html + // - logs-* + // - data-rolling + // PSEUDO: Create 5 indices on the cluster + // - logs-01-2345 + // - logs-12-3456 + // - data-rolling + // - playground + // - playground2 + // PSEUDO: Add documents + // - 19x http-data docs into logs-01-2345 + // - 23x http-data docs into logs-12-3456 + // - 29x data-rolling + // - 5x geonames docs into playground + // - 7x geopoint into playground2 + + // PSEUDO: Create a target cluster running OS 2.X (Where x is the latest released version) + + // Action + // PSEUDO: Migrate from the snapshot + // simpleRfsInstance.fullMigrationViaLocalSnapshot(targetCluster.toString()); + // PSEUDO: Shutdown source cluster + + // Validation + + // PSEUDO: Verify creation of 2 index templates on the clustqer + // PSEUDO: Verify creation of 5 indices on the cluster + // - logs-01-2345 + // - logs-12-3456 + // - data-rolling + // - playground + // - playground2 + // PSEUDO: Verify documents + + // PSEUDO: Additional validation: + // - Mapping type parameter is removed + // } } From bf749f882b754713a99ede0695cf7d90298a8557 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 5 Jun 2024 20:57:15 +0000 Subject: [PATCH 10/10] Special case removal of mapping types Signed-off-by: Peter Nied --- RFS/src/test/java/com/rfs/integration/EndToEndTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java index 8dc6d24f9..f41175df3 100644 --- a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java +++ b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java @@ -1,12 +1,12 @@ package com.rfs.integration; import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; import com.rfs.framework.ElasticsearchContainer; import com.rfs.framework.OpenSearchContainer; +import com.rfs.framework.OpenSearchContainer.Version; import com.rfs.framework.SimpleRestoreFromSnapshot; import lombok.extern.slf4j.Slf4j; @@ -24,7 +24,7 @@ public class EndToEndTest { @ParameterizedTest(name = "Target OpenSearch {0}") @ArgumentsSource(SupportedTargetCluster.class) @Disabled - public void ES_v6_8_Migration(final OpenSearchContainer.Version targetVersion) throws Exception { + public void migrateFrom_ES_v6_8(final OpenSearchContainer.Version targetVersion) throws Exception { // Setup // PSEUDO: Create a source cluster running ES 6.8 // PSEUDO: Create 2 templates on the cluster, see https://www.elastic.co/guide/en/elasticsearch/reference/6.8/indices-templates.html @@ -62,8 +62,9 @@ public void ES_v6_8_Migration(final OpenSearchContainer.Version targetVersion) t // PSEUDO: Verify documents // PSEUDO: Additional validation: - // - Mapping type parameter is removed - // + if (targetVersion == Version.V2_14_0) { + // - Mapping type parameter is removed https://opensearch.org/docs/latest/breaking-changes/#remove-mapping-types-parameter + } } @ParameterizedTest(name = "Target OpenSearch {0}")