From bfa6c9235baad6f20518925deb807c9dd6a7ef12 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 1 Aug 2024 20:12:26 +0000 Subject: [PATCH] Passing test cases Signed-off-by: Peter Nied --- .../migrations/MetadataMigration.java | 3 + .../migrations/commands/Migrate.java | 26 ++-- .../migrations/commands/Result.java | 1 + .../opensearch/migrations/EndToEndTest.java | 112 ++++++++++++++++++ .../migrations/commands/MigrateTest.java | 23 ++++ .../com/rfs/integration/EndToEndTest.java | 2 +- .../rfs/integration/SnapshotStateTest.java | 3 +- .../java/com/rfs/http}/ClusterOperations.java | 2 +- 8 files changed, 159 insertions(+), 13 deletions(-) create mode 100644 MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java create mode 100644 MetadataMigration/src/test/java/org/opensearch/migrations/commands/MigrateTest.java rename RFS/src/{test/java/com/rfs/framework => testFixtures/java/com/rfs/http}/ClusterOperations.java (99%) diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataMigration.java b/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataMigration.java index 03b4e1168..cc0e6899e 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataMigration.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/MetadataMigration.java @@ -14,11 +14,14 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.ParametersDelegate; import com.rfs.common.ConnectionDetails; + +import lombok.ToString; import lombok.extern.slf4j.Slf4j; @Slf4j public class MetadataMigration { + @ToString public static class Args { @Parameter(names = { "--snapshot-name" }, description = "The name of the snapshot to migrate", required = true) public String snapshotName; diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java index 68ee7e05a..5a93cd835 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java @@ -32,7 +32,6 @@ @Slf4j public class Migrate { - private final Args arguments; public Migrate(Args arguments) { @@ -40,15 +39,20 @@ public Migrate(Args arguments) { } public MigrateResult execute(RootMetadataMigrationContext context) { - - if (arguments.fileSystemRepoPath == null && arguments.s3RepoUri == null) { - throw new ParameterException("Either file-system-repo-path or s3-repo-uri must be set"); - } - if (arguments.fileSystemRepoPath != null && arguments.s3RepoUri != null) { - throw new ParameterException("Only one of file-system-repo-path and s3-repo-uri can be set"); - } - if ((arguments.s3RepoUri != null) && (arguments.s3Region == null || arguments.s3LocalDirPath == null)) { - throw new ParameterException("If an s3 repo is being used, s3-region and s3-local-dir-path must be set"); + log.atInfo().setMessage("Command line arguments {0}").addArgument(arguments::toString).log(); + try { + if (arguments.fileSystemRepoPath == null && arguments.s3RepoUri == null) { + throw new ParameterException("Either file-system-repo-path or s3-repo-uri must be set"); + } + if (arguments.fileSystemRepoPath != null && arguments.s3RepoUri != null) { + throw new ParameterException("Only one of file-system-repo-path and s3-repo-uri can be set"); + } + if ((arguments.s3RepoUri != null) && (arguments.s3Region == null || arguments.s3LocalDirPath == null)) { + throw new ParameterException("If an s3 repo is being used, s3-region and s3-local-dir-path must be set"); + } + } catch (Exception e) { + log.atError().setMessage("Invalid parameter").setCause(e).log(); + return new MigrateResult(1); } final String snapshotName = arguments.snapshotName; @@ -87,6 +91,7 @@ public MigrateResult execute(RootMetadataMigrationContext context) { awarenessDimensionality ); new MetadataRunner(snapshotName, metadataFactory, metadataCreator, transformer).migrateMetadata(); + log.info("Metadata copy complete."); final IndexMetadata.Factory indexMetadataFactory = new IndexMetadataFactory_ES_7_10(repoDataProvider); final IndexCreator_OS_2_11 indexCreator = new IndexCreator_OS_2_11(targetClient); @@ -98,6 +103,7 @@ public MigrateResult execute(RootMetadataMigrationContext context) { indexAllowlist, context.createIndexContext() ).migrateIndices(); + log.info("Index copy complete."); } catch (Exception e) { log.atError().setMessage("Unexpected failure").setCause(e).log(); return new MigrateResult(1); diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Result.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Result.java index 91b966813..552d706fb 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Result.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Result.java @@ -1,5 +1,6 @@ package org.opensearch.migrations.commands; +/** All shared cli result information */ public interface Result { int getExitCode(); } diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java new file mode 100644 index 000000000..d73309e93 --- /dev/null +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java @@ -0,0 +1,112 @@ +package org.opensearch.migrations; + +import java.io.File; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.opensearch.migrations.MetadataMigration.Args; +import org.opensearch.migrations.metadata.tracing.MetadataMigrationTestContext; +import org.opensearch.migrations.snapshot.creation.tracing.SnapshotTestContext; + +import com.rfs.common.FileSystemSnapshotCreator; +import com.rfs.common.OpenSearchClient; +import com.rfs.common.ConnectionDetails.TargetArgs; +import com.rfs.framework.SearchClusterContainer; +import com.rfs.http.ClusterOperations; +import com.rfs.worker.SnapshotRunner; +import lombok.SneakyThrows; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Tests focused on setting up whole source clusters, performing a migration, and validation on the target cluster + */ +class EndToEndTest { + + @TempDir + private File localDirectory; + + @Test + void metadataMigrateFrom_ES_v7_10_to_OS_v2_14() throws Exception { + final var targetVersion = SearchClusterContainer.OS_V2_14_0; + try ( + final var sourceCluster = new SearchClusterContainer(SearchClusterContainer.ES_V7_10_2); + final var targetCluster = new SearchClusterContainer(targetVersion) + ) { + migrateFrom_ES_v7_X(sourceCluster, targetCluster); + } + } + + @SneakyThrows + private void migrateFrom_ES_v7_X( + final SearchClusterContainer sourceCluster, + final SearchClusterContainer targetCluster + ) { + // ACTION: Set up the source/target clusters + var bothClustersStarted = CompletableFuture.allOf( + CompletableFuture.runAsync(() -> sourceCluster.start()), + CompletableFuture.runAsync(() -> targetCluster.start()) + ); + bothClustersStarted.join(); + + // Create the component and index templates + var sourceClusterOperations = new ClusterOperations(sourceCluster.getUrl()); + var compoTemplateName = "simple_component_template"; + var indexTemplateName = "simple_index_template"; + sourceClusterOperations.createES7Templates(compoTemplateName, indexTemplateName, "author", "blog*"); + + // Creates a document that uses the template + var indexName = "blog_2023"; + sourceClusterOperations.createDocument(indexName, "222", "{\"author\":\"Tobias Funke\"}"); + + // ACTION: Take a snapshot + var snapshotContext = SnapshotTestContext.factory().noOtelTracking(); + var snapshotName = "my_snap"; + var sourceClient = new OpenSearchClient(sourceCluster.getUrl(), null, null, true); + var snapshotCreator = new FileSystemSnapshotCreator( + snapshotName, + sourceClient, + SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, + snapshotContext.createSnapshotCreateContext() + ); + SnapshotRunner.runAndWaitForCompletion(snapshotCreator); + sourceCluster.copySnapshotData(localDirectory.toString()); + + var targetArgs = new TargetArgs(); + targetArgs.host = targetCluster.getUrl(); + + var arguments = new Args(); + arguments.fileSystemRepoPath = localDirectory.getAbsolutePath(); + arguments.snapshotName = snapshotName; + arguments.targetArgs = targetArgs; + arguments.indexAllowlist = List.of(indexName); + arguments.componentTemplateAllowlist = List.of(compoTemplateName); + arguments.indexTemplateAllowlist = List.of(indexTemplateName); + + // ACTION: Migrate the templates + var metadataContext = MetadataMigrationTestContext.factory().noOtelTracking(); + var result = new MetadataMigration(arguments).migrate().execute(metadataContext); + + assertThat(result.getExitCode(), equalTo(0)); + + // Check that the templates were migrated + var targetClusterOperations = new ClusterOperations(targetCluster.getUrl()); + var res = targetClusterOperations.get("/_index_template/" + indexTemplateName); + assertThat(res.getValue(), res.getKey(), equalTo(200)); + assertThat(res.getValue(), Matchers.containsString("composed_of\":[\"" + compoTemplateName + "\"]")); + + // Check that the index was migrated + res = targetClusterOperations.get("/" + indexName); + assertThat(res.getValue(), res.getKey(), equalTo(200)); + + // PSEUDO: Additional validation: + if (SearchClusterContainer.OS_V2_14_0.equals(targetCluster.getVersion())) { + // - Mapping type parameter is removed + // https://opensearch.org/docs/latest/breaking-changes/#remove-mapping-types-parameter + } + } +} diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/commands/MigrateTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/commands/MigrateTest.java new file mode 100644 index 000000000..bcea112e2 --- /dev/null +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/commands/MigrateTest.java @@ -0,0 +1,23 @@ +package org.opensearch.migrations.commands; + +import org.junit.jupiter.api.Test; + +import org.opensearch.migrations.MetadataMigration; +import org.opensearch.migrations.metadata.tracing.RootMetadataMigrationContext; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.Mockito.mock; + +class MigrateTest { + + @Test + void configureSource_noSourceSet() { + var meta = new MetadataMigration(mock(MetadataMigration.Args.class)); + + var configureSource = meta.migrate() + .execute(mock(RootMetadataMigrationContext.class)); + + assertThat(configureSource.getExitCode(), equalTo(1)); + } +} diff --git a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java index f79a46f9e..099d532e3 100644 --- a/RFS/src/test/java/com/rfs/integration/EndToEndTest.java +++ b/RFS/src/test/java/com/rfs/integration/EndToEndTest.java @@ -16,9 +16,9 @@ import com.rfs.common.FileSystemRepo; import com.rfs.common.FileSystemSnapshotCreator; import com.rfs.common.OpenSearchClient; -import com.rfs.framework.ClusterOperations; import com.rfs.framework.SearchClusterContainer; import com.rfs.framework.SimpleRestoreFromSnapshot; +import com.rfs.http.ClusterOperations; import com.rfs.transformers.TransformFunctions; import com.rfs.version_es_6_8.GlobalMetadataFactory_ES_6_8; import com.rfs.version_es_6_8.IndexMetadataFactory_ES_6_8; diff --git a/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java b/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java index 0a38d1a3f..f8bf5c776 100644 --- a/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java +++ b/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java @@ -13,9 +13,10 @@ import org.opensearch.migrations.workcoordination.tracing.WorkCoordinationTestContext; import com.rfs.common.OpenSearchClient; -import com.rfs.framework.ClusterOperations; import com.rfs.framework.SearchClusterContainer; import com.rfs.framework.SimpleRestoreFromSnapshot_ES_7_10; +import com.rfs.http.ClusterOperations; + import org.mockito.ArgumentCaptor; import reactor.core.publisher.Mono; diff --git a/RFS/src/test/java/com/rfs/framework/ClusterOperations.java b/RFS/src/testFixtures/java/com/rfs/http/ClusterOperations.java similarity index 99% rename from RFS/src/test/java/com/rfs/framework/ClusterOperations.java rename to RFS/src/testFixtures/java/com/rfs/http/ClusterOperations.java index c091ce4a4..ebb7addd3 100644 --- a/RFS/src/test/java/com/rfs/framework/ClusterOperations.java +++ b/RFS/src/testFixtures/java/com/rfs/http/ClusterOperations.java @@ -1,4 +1,4 @@ -package com.rfs.framework; +package com.rfs.http; import java.io.IOException; import java.nio.charset.StandardCharsets;