Merge pull request #694 from peternied/e2e-scenarios

Define end to end scenario tests

RFS/src/test/java/com/rfs/framework/OpenSearchContainer.java

@@ -0,0 +1,65 @@
+package com.rfs.framework;
+
+import java.time.Duration;
+
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.DockerImageName;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Containerized version of OpenSearch cluster
+ */
+@Slf4j
+public class OpenSearchContainer implements AutoCloseable {
+
+    private final GenericContainer<?> container;
+    private final Version version;
+
+    @SuppressWarnings("resource")
+    public OpenSearchContainer(final Version version) {
+        this.version = version;
+        container = new GenericContainer<>(DockerImageName.parse(this.version.imageName))
+                .withExposedPorts(9200)
+                .withEnv("discovery.type", "single-node")
+                .withEnv("plugins.security.disabled", "true")
+                .withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "SecurityIsDisabled123$%^")
+                .waitingFor(Wait.forHttp("/").forPort(9200).forStatusCode(200).withStartupTimeout(Duration.ofMinutes(1)));
+    }
+
+    public String getUrl() {
+        final var address = container.getHost();
+        final var port = container.getMappedPort(9200);
+        return "http://" + address + ":" + port;
+    }
+
+    public void start() {
+        log.info("Starting version: " + version);
+        container.start();
+    }
+
+    @Override
+    public void close() throws Exception {
+        log.info("Stopping version: " + version);
+        log.debug("Instance logs:\n" + container.getLogs());
+        container.stop();
+    }
+
+    public static enum Version {
+        V1_3_15("opensearchproject/opensearch:1.3.16", "1.3.16"),
+        V2_14_0("opensearchproject/opensearch:2.14.0", "2.14.0");
+
+        final String imageName;
+        final String prettyName;
+        Version(final String imageName, final String prettyName) {
+            this.imageName = imageName;
+            this.prettyName = prettyName;
+        }
+
+        @Override
+        public String toString() {
+            return prettyName;
+        }
+    }
+}

RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot.java

@@ -0,0 +1,38 @@
+package com.rfs.framework;
+
+import java.nio.file.Path;
+import java.util.List;
+
+import com.rfs.common.ConnectionDetails;
+import com.rfs.common.IndexMetadata;
+import com.rfs.common.OpenSearchClient;
+
+public interface SimpleRestoreFromSnapshot {
+
+    public static SimpleRestoreFromSnapshot forCluster(final String sourceClusterUrl) {
+        // TODO: determine version from source cluster
+        return new SimpleRestoreFromSnapshot_ES_7_10();
+    }
+
+    public default void fullMigrationViaLocalSnapshot(final String targetClusterUrl) throws Exception {
+        // TODO: Dynamically create / clean these up during tests
+        final var tempSnapshotName = "";
+        final var compressedSnapshotDirectory = "";
+        final var unpackedShardDataDir = Path.of("");
+        final var indices = extractSnapshotIndexData(compressedSnapshotDirectory, tempSnapshotName, unpackedShardDataDir);
+        final var targetClusterClient = new OpenSearchClient(new ConnectionDetails(targetClusterUrl, null, null)); 
+
+        // TODO: This should update the following metdata:
+        //    - Global cluster state
+        //    - Index Templates
+        //    - Indices
+        //    - Documents
+
+        updateTargetCluster(indices, unpackedShardDataDir, targetClusterClient);
+    }
+
+    public List<IndexMetadata.Data> extractSnapshotIndexData(final String localPath, final String snapshotName, final Path unpackedShardDataDir) throws Exception;
+
+    public void updateTargetCluster(final List<IndexMetadata.Data> indices, final Path unpackedShardDataDir, final OpenSearchClient client) throws Exception;
+
+}

RFS/src/test/java/com/rfs/framework/SimpleRestoreFromSnapshot_ES_7_10.java

@@ -22,11 +22,11 @@
 /**
  * Simplified version of RFS for use in testing - ES 7.10 version.
  */
-public class SimpleRestoreFromSnapshot_ES_7_10 {
+public class SimpleRestoreFromSnapshot_ES_7_10 implements SimpleRestoreFromSnapshot {
 
     private static final Logger logger = LogManager.getLogger(SimpleRestoreFromSnapshot_ES_7_10.class);
 
-    public List<IndexMetadata.Data> extraSnapshotIndexData(final String localPath, final String snapshotName, final Path unpackedShardDataDir) throws Exception {
+    public List<IndexMetadata.Data> extractSnapshotIndexData(final String localPath, final String snapshotName, final Path unpackedShardDataDir) throws Exception {
         IOUtils.rm(unpackedShardDataDir);
 
         final var repo = new FileSystemRepo(Path.of(localPath));

RFS/src/test/java/com/rfs/integration/EndToEndTest.java

@@ -0,0 +1,129 @@
+package com.rfs.integration;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ArgumentsSource;
+
+import com.rfs.framework.ElasticsearchContainer;
+import com.rfs.framework.OpenSearchContainer;
+import com.rfs.framework.OpenSearchContainer.Version;
+import com.rfs.framework.SimpleRestoreFromSnapshot;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Tests focused on setting up whole source clusters, performing a migration, and validation on the target cluster
+ */
+@Slf4j
+public class EndToEndTest {
+
+    protected Object sourceCluster;
+    protected Object targetCluster;
+    protected SimpleRestoreFromSnapshot simpleRfsInstance;
+
+    @ParameterizedTest(name = "Target OpenSearch {0}")
+    @ArgumentsSource(SupportedTargetCluster.class)
+    @Disabled
+    public void migrateFrom_ES_v6_8(final OpenSearchContainer.Version targetVersion) throws Exception {
+        // Setup
+        // PSEUDO: Create a source cluster running ES 6.8
+        // PSEUDO: Create 2 templates on the cluster, see https://www.elastic.co/guide/en/elasticsearch/reference/6.8/indices-templates.html
+        //    - logs-*
+        //    - data-rolling
+        // PSEUDO: Create 5 indices on the cluster
+        //    - logs-01-2345
+        //    - logs-12-3456
+        //    - data-rolling
+        //    - playground
+        //    - playground2
+        // PSEUDO: Add documents
+        //    - 19x http-data docs into logs-01-2345
+        //    - 23x http-data docs into logs-12-3456
+        //    - 29x data-rolling
+        //    - 5x geonames docs into playground
+        //    - 7x geopoint into playground2
+
+        // PSEUDO: Create a target cluster running OS 2.X (Where x is the latest released version)
+
+        // Action
+        // PSEUDO: Migrate from the snapshot
+        // simpleRfsInstance.fullMigrationViaLocalSnapshot(targetCluster.toString());
+        // PSEUDO: Shutdown source cluster
+
+        // Validation
+
+        // PSEUDO: Verify creation of 2 index templates on the cluster
+        // PSEUDO: Verify creation of 5 indices on the cluster
+        //    - logs-01-2345
+        //    - logs-12-3456
+        //    - data-rolling
+        //    - playground
+        //    - playground2
+        // PSEUDO: Verify documents
+
+        // PSEUDO: Additional validation:
+        if (targetVersion == Version.V2_14_0) {
+            //   - Mapping type parameter is removed https://opensearch.org/docs/latest/breaking-changes/#remove-mapping-types-parameter
+        }
+    }
+
+    @ParameterizedTest(name = "Target OpenSearch {0}")
+    @ArgumentsSource(SupportedTargetCluster.class)
+    @Disabled
+    public void migrateFrom_ES_v7_10(final OpenSearchContainer.Version targetVersion) throws Exception {
+        // Setup
+        // PSEUDO: Create a source cluster running ES 6.8
+
+        migrateFrom_ES_v7_X(null);
+    }
+
+    @ParameterizedTest(name = "Target OpenSearch {0}")
+    @ArgumentsSource(SupportedTargetCluster.class)
+    @Disabled
+    public void migrateFrom_ES_v7_17(final OpenSearchContainer.Version targetVersion) throws Exception {
+        // Setup
+        // PSEUDO: Create a source cluster running ES 6.8
+
+        migrateFrom_ES_v7_X(null);
+    }
+
+    private void migrateFrom_ES_v7_X(final ElasticsearchContainer sourceCluster) {
+        // PSEUDO: Create 2 index templates on the cluster, see https://www.elastic.co/guide/en/elasticsearch/reference/7.17/index-templates.html
+        //    - logs-*
+        //    - data-rolling
+        // PSEUDO: Create 5 indices on the cluster
+        //    - logs-01-2345
+        //    - logs-12-3456
+        //    - data-rolling
+        //    - playground
+        //    - playground2
+        // PSEUDO: Add documents
+        //    - 19x http-data docs into logs-01-2345
+        //    - 23x http-data docs into logs-12-3456
+        //    - 29x data-rolling
+        //    - 5x geonames docs into playground
+        //    - 7x geopoint into playground2
+
+        // PSEUDO: Create a target cluster running OS 2.X (Where x is the latest released version)
+
+        // Action
+        // PSEUDO: Migrate from the snapshot
+        // simpleRfsInstance.fullMigrationViaLocalSnapshot(targetCluster.toString());
+        // PSEUDO: Shutdown source cluster
+
+        // Validation
+
+        // PSEUDO: Verify creation of 2 index templates on the clustqer
+        // PSEUDO: Verify creation of 5 indices on the cluster
+        //    - logs-01-2345
+        //    - logs-12-3456
+        //    - data-rolling
+        //    - playground
+        //    - playground2
+        // PSEUDO: Verify documents
+
+        // PSEUDO: Additional validation:
+        //   - Mapping type parameter is removed
+        //
+    }
+}

RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java

@@ -72,7 +72,7 @@ public void SingleSnapshot_SingleDocument() throws Exception {
         cluster.copySnapshotData(snapshotCopy.getAbsolutePath());
 
         final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data");
-        final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir);
+        final var indices = srfs.extractSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir);
 
         final var client = mock(OpenSearchClient.class);
         when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty());
@@ -105,7 +105,7 @@ public void SingleSnapshot_SingleDocument_Then_DeletedDocument() throws Exceptio
         cluster.copySnapshotData(snapshotCopy.getAbsolutePath());
 
         final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data");
-        final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir);
+        final var indices = srfs.extractSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir);
 
         final var client = mock(OpenSearchClient.class);
         when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty());
@@ -140,7 +140,7 @@ public void SingleSnapshot_SingleDocument_Then_UpdateDocument() throws Exception
         cluster.copySnapshotData(snapshotCopy.getAbsolutePath());
 
         final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data");
-        final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir);
+        final var indices = srfs.extractSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir);
 
         final var client = mock(OpenSearchClient.class);
         when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty());

RFS/src/test/java/com/rfs/integration/SupportedTargetCluster.java

@@ -0,0 +1,22 @@
+package com.rfs.integration;
+
+import java.util.stream.Stream;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.ArgumentsProvider;
+
+import com.rfs.framework.OpenSearchContainer;
+
+/**
+ * Defines all supported target clusters
+ */
+public class SupportedTargetCluster implements ArgumentsProvider {
+
+    @Override
+    public Stream<? extends Arguments> provideArguments(final ExtensionContext context) {
+        return Stream.of(
+                Arguments.of(OpenSearchContainer.Version.V1_3_15), 
+                Arguments.of(OpenSearchContainer.Version.V2_14_0)
+        );
+    }
+}