diff --git a/RFS/src/test/java/com/rfs/framework/OpenSearchContainer.java b/RFS/src/test/java/com/rfs/framework/OpenSearchContainer.java new file mode 100644 index 000000000..7a2241c1b --- /dev/null +++ b/RFS/src/test/java/com/rfs/framework/OpenSearchContainer.java @@ -0,0 +1,65 @@ +package com.rfs.framework; + +import java.time.Duration; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +import lombok.extern.slf4j.Slf4j; + +/** + * Containerized version of OpenSearch cluster + */ +@Slf4j +public class OpenSearchContainer implements AutoCloseable { + + private final GenericContainer container; + private final Version version; + + @SuppressWarnings("resource") + public OpenSearchContainer(final Version version) { + this.version = version; + container = new GenericContainer<>(DockerImageName.parse(this.version.imageName)) + .withExposedPorts(9200) + .withEnv("discovery.type", "single-node") + .withEnv("plugins.security.disabled", "true") + .withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "SecurityIsDisabled123$%^") + .waitingFor(Wait.forHttp("/").forPort(9200).forStatusCode(200).withStartupTimeout(Duration.ofMinutes(1))); + } + + public String getUrl() { + final var address = container.getHost(); + final var port = container.getMappedPort(9200); + return "http://" + address + ":" + port; + } + + public void start() { + log.info("Starting version: " + version); + container.start(); + } + + @Override + public void close() throws Exception { + log.info("Stopping version: " + version); + log.debug("Instance logs:\n" + container.getLogs()); + container.stop(); + } + + public static enum Version { + V1_3_15("opensearchproject/opensearch:1.3.16", "1.3.16"), + V2_14_0("opensearchproject/opensearch:2.14.0", "2.14.0"); + + final String imageName; + final String prettyName; + Version(final String imageName, final String prettyName) { + this.imageName = imageName; + this.prettyName = prettyName; + } + + @Override + public String toString() { + return prettyName; + } + } +} diff --git a/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot.java b/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot.java new file mode 100644 index 000000000..9fd74a0e6 --- /dev/null +++ b/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot.java @@ -0,0 +1,38 @@ +package com.rfs.framework; + +import java.nio.file.Path; +import java.util.List; + +import com.rfs.common.ConnectionDetails; +import com.rfs.common.IndexMetadata; +import com.rfs.common.OpenSearchClient; + +public interface SimpleRestoreFromSnapshot { + + public static SimpleRestoreFromSnapshot forCluster(final String sourceClusterUrl) { + // TODO: determine version from source cluster + return new SimpleRestoreFromSnapshot_ES_7_10(); + } + + public default void fullMigrationViaLocalSnapshot(final String targetClusterUrl) throws Exception { + // TODO: Dynamically create / clean these up during tests + final var tempSnapshotName = ""; + final var compressedSnapshotDirectory = ""; + final var unpackedShardDataDir = Path.of(""); + final var indices = extractSnapshotIndexData(compressedSnapshotDirectory, tempSnapshotName, unpackedShardDataDir); + final var targetClusterClient = new OpenSearchClient(new ConnectionDetails(targetClusterUrl, null, null)); + + // TODO: This should update the following metdata: + // - Global cluster state + // - Index Templates + // - Indices + // - Documents + + updateTargetCluster(indices, unpackedShardDataDir, targetClusterClient); + } + + public List extractSnapshotIndexData(final String localPath, final String snapshotName, final Path unpackedShardDataDir) throws Exception; + + public void updateTargetCluster(final List indices, final Path unpackedShardDataDir, final OpenSearchClient client) throws Exception; + +} diff --git a/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot_ES_7_10.java b/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot_ES_7_10.java index 5ca87ee1a..df26fad1c 100644 --- a/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot_ES_7_10.java +++ b/RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot_ES_7_10.java @@ -22,11 +22,11 @@ /** * Simplified version of RFS for use in testing - ES 7.10 version. */ -public class SimpleRestoreFromSnapshot_ES_7_10 { +public class SimpleRestoreFromSnapshot_ES_7_10 implements SimpleRestoreFromSnapshot { private static final Logger logger = LogManager.getLogger(SimpleRestoreFromSnapshot_ES_7_10.class); - public List extraSnapshotIndexData(final String localPath, final String snapshotName, final Path unpackedShardDataDir) throws Exception { + public List extractSnapshotIndexData(final String localPath, final String snapshotName, final Path unpackedShardDataDir) throws Exception { IOUtils.rm(unpackedShardDataDir); final var repo = new FileSystemRepo(Path.of(localPath)); diff --git a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java new file mode 100644 index 000000000..f41175df3 --- /dev/null +++ b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java @@ -0,0 +1,129 @@ +package com.rfs.integration; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ArgumentsSource; + +import com.rfs.framework.ElasticsearchContainer; +import com.rfs.framework.OpenSearchContainer; +import com.rfs.framework.OpenSearchContainer.Version; +import com.rfs.framework.SimpleRestoreFromSnapshot; + +import lombok.extern.slf4j.Slf4j; + +/** + * Tests focused on setting up whole source clusters, performing a migration, and validation on the target cluster + */ +@Slf4j +public class EndToEndTest { + + protected Object sourceCluster; + protected Object targetCluster; + protected SimpleRestoreFromSnapshot simpleRfsInstance; + + @ParameterizedTest(name = "Target OpenSearch {0}") + @ArgumentsSource(SupportedTargetCluster.class) + @Disabled + public void migrateFrom_ES_v6_8(final OpenSearchContainer.Version targetVersion) throws Exception { + // Setup + // PSEUDO: Create a source cluster running ES 6.8 + // PSEUDO: Create 2 templates on the cluster, see https://www.elastic.co/guide/en/elasticsearch/reference/6.8/indices-templates.html + // - logs-* + // - data-rolling + // PSEUDO: Create 5 indices on the cluster + // - logs-01-2345 + // - logs-12-3456 + // - data-rolling + // - playground + // - playground2 + // PSEUDO: Add documents + // - 19x http-data docs into logs-01-2345 + // - 23x http-data docs into logs-12-3456 + // - 29x data-rolling + // - 5x geonames docs into playground + // - 7x geopoint into playground2 + + // PSEUDO: Create a target cluster running OS 2.X (Where x is the latest released version) + + // Action + // PSEUDO: Migrate from the snapshot + // simpleRfsInstance.fullMigrationViaLocalSnapshot(targetCluster.toString()); + // PSEUDO: Shutdown source cluster + + // Validation + + // PSEUDO: Verify creation of 2 index templates on the cluster + // PSEUDO: Verify creation of 5 indices on the cluster + // - logs-01-2345 + // - logs-12-3456 + // - data-rolling + // - playground + // - playground2 + // PSEUDO: Verify documents + + // PSEUDO: Additional validation: + if (targetVersion == Version.V2_14_0) { + // - Mapping type parameter is removed https://opensearch.org/docs/latest/breaking-changes/#remove-mapping-types-parameter + } + } + + @ParameterizedTest(name = "Target OpenSearch {0}") + @ArgumentsSource(SupportedTargetCluster.class) + @Disabled + public void migrateFrom_ES_v7_10(final OpenSearchContainer.Version targetVersion) throws Exception { + // Setup + // PSEUDO: Create a source cluster running ES 6.8 + + migrateFrom_ES_v7_X(null); + } + + @ParameterizedTest(name = "Target OpenSearch {0}") + @ArgumentsSource(SupportedTargetCluster.class) + @Disabled + public void migrateFrom_ES_v7_17(final OpenSearchContainer.Version targetVersion) throws Exception { + // Setup + // PSEUDO: Create a source cluster running ES 6.8 + + migrateFrom_ES_v7_X(null); + } + + private void migrateFrom_ES_v7_X(final ElasticsearchContainer sourceCluster) { + // PSEUDO: Create 2 index templates on the cluster, see https://www.elastic.co/guide/en/elasticsearch/reference/7.17/index-templates.html + // - logs-* + // - data-rolling + // PSEUDO: Create 5 indices on the cluster + // - logs-01-2345 + // - logs-12-3456 + // - data-rolling + // - playground + // - playground2 + // PSEUDO: Add documents + // - 19x http-data docs into logs-01-2345 + // - 23x http-data docs into logs-12-3456 + // - 29x data-rolling + // - 5x geonames docs into playground + // - 7x geopoint into playground2 + + // PSEUDO: Create a target cluster running OS 2.X (Where x is the latest released version) + + // Action + // PSEUDO: Migrate from the snapshot + // simpleRfsInstance.fullMigrationViaLocalSnapshot(targetCluster.toString()); + // PSEUDO: Shutdown source cluster + + // Validation + + // PSEUDO: Verify creation of 2 index templates on the clustqer + // PSEUDO: Verify creation of 5 indices on the cluster + // - logs-01-2345 + // - logs-12-3456 + // - data-rolling + // - playground + // - playground2 + // PSEUDO: Verify documents + + // PSEUDO: Additional validation: + // - Mapping type parameter is removed + // + } +} diff --git a/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java b/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java index 39c1f9d47..470de3c5c 100644 --- a/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java +++ b/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java @@ -72,7 +72,7 @@ public void SingleSnapshot_SingleDocument() throws Exception { cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data"); - final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); + final var indices = srfs.extractSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); final var client = mock(OpenSearchClient.class); when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty()); @@ -105,7 +105,7 @@ public void SingleSnapshot_SingleDocument_Then_DeletedDocument() throws Exceptio cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data"); - final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); + final var indices = srfs.extractSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); final var client = mock(OpenSearchClient.class); when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty()); @@ -140,7 +140,7 @@ public void SingleSnapshot_SingleDocument_Then_UpdateDocument() throws Exception cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data"); - final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); + final var indices = srfs.extractSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); final var client = mock(OpenSearchClient.class); when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty()); diff --git a/RFS/src/test/java/com/rfs/integration/SupportedTargetCluster.java b/RFS/src/test/java/com/rfs/integration/SupportedTargetCluster.java new file mode 100644 index 000000000..a8d9d511e --- /dev/null +++ b/RFS/src/test/java/com/rfs/integration/SupportedTargetCluster.java @@ -0,0 +1,22 @@ +package com.rfs.integration; + +import java.util.stream.Stream; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; + +import com.rfs.framework.OpenSearchContainer; + +/** + * Defines all supported target clusters + */ +public class SupportedTargetCluster implements ArgumentsProvider { + + @Override + public Stream provideArguments(final ExtensionContext context) { + return Stream.of( + Arguments.of(OpenSearchContainer.Version.V1_3_15), + Arguments.of(OpenSearchContainer.Version.V2_14_0) + ); + } +} \ No newline at end of file