From e9a5567f42d6e4741eb646177e63352631ae4476 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 28 May 2024 17:25:40 +0000 Subject: [PATCH] DCopy snapshot data off of container Signed-off-by: Peter Nied --- .../com/rfs/framework/ClusterOperations.java | 2 +- .../rfs/framework/ElasticsearchContainer.java | 33 ++++++++++++++--- .../rfs/integration/SnapshotStateTest.java | 36 ++++++++++--------- 3 files changed, 50 insertions(+), 21 deletions(-) diff --git a/RFS/src/test/java/com/rfs/framework/ClusterOperations.java b/RFS/src/test/java/com/rfs/framework/ClusterOperations.java index a8cea9e18..0f09ad5ff 100644 --- a/RFS/src/test/java/com/rfs/framework/ClusterOperations.java +++ b/RFS/src/test/java/com/rfs/framework/ClusterOperations.java @@ -30,7 +30,7 @@ public void createSnapshotRepository() throws IOException { final var repositoryJson = "{\n" + " \"type\": \"fs\",\n" + " \"settings\": {\n" + - " \"location\": \"/snapshots\",\n" + + " \"location\": \"/usr/share/elasticsearch/snapshots\",\n" + " \"compress\": false\n" + " }\n" + "}"; diff --git a/RFS/src/test/java/com/rfs/framework/ElasticsearchContainer.java b/RFS/src/test/java/com/rfs/framework/ElasticsearchContainer.java index 72891eeeb..0d1c88679 100644 --- a/RFS/src/test/java/com/rfs/framework/ElasticsearchContainer.java +++ b/RFS/src/test/java/com/rfs/framework/ElasticsearchContainer.java @@ -1,10 +1,10 @@ package com.rfs.framework; +import java.io.File; import java.time.Duration; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.utility.DockerImageName; @@ -15,21 +15,46 @@ public class ElasticsearchContainer implements AutoCloseable { private static final Logger logger = LogManager.getLogger(ElasticsearchContainer.class); + private static final String CLUSTER_SNAPSHOT_DIR = "/usr/share/elasticsearch/snapshots"; private final GenericContainer container; private final Version version; @SuppressWarnings("resource") - public ElasticsearchContainer(final Version version, final String localSnapshotDirectory) { + public ElasticsearchContainer(final Version version) { this.version = version; container = new GenericContainer<>(DockerImageName.parse(this.version.imageName)) .withExposedPorts(9200, 9300) .withEnv("discovery.type", "single-node") - .withEnv("path.repo", "/snapshots") - .withFileSystemBind(localSnapshotDirectory, "/snapshots", BindMode.READ_WRITE) + .withEnv("path.repo", CLUSTER_SNAPSHOT_DIR) .waitingFor(Wait.forHttp("/").forPort(9200).forStatusCode(200).withStartupTimeout(Duration.ofMinutes(1))); } + public void copySnapshotData(final String directory) { + logger.info("Copy stuff was called"); + try { + // Execute command to list all files in the directory + final var result = container.execInContainer("sh", "-c", "find " + CLUSTER_SNAPSHOT_DIR + " -type f"); + logger.debug("Process Exit Code: " + result.getExitCode()); + logger.debug("Standard Output: " + result.getStdout()); + logger.debug("Standard Error : " + result.getStderr()); + // Process each file and copy it from the container + try (final var lines = result.getStdout().lines()) { + lines.forEach(fullFilePath -> { + final var file = fullFilePath.substring(CLUSTER_SNAPSHOT_DIR.length() + 1); + final var sourcePath = CLUSTER_SNAPSHOT_DIR + "/" + file; + final var destinationPath = directory + "/" + file; + // Make sure the parent directory tree exists before copying + new File(destinationPath).getParentFile().mkdirs(); + logger.info("Copying file " + sourcePath + " from container onto " + destinationPath); + container.copyFileFromContainer(sourcePath, destinationPath); + }); + } + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + public void start() { logger.info("Starting ElasticsearchContainer version:" + version.prettyName); container.start(); diff --git a/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java b/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java index 8d75c9f74..39c1f9d47 100644 --- a/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java +++ b/RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java @@ -1,10 +1,10 @@ package com.rfs.integration; -import org.apache.lucene.util.IOUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.mockito.ArgumentCaptor; import com.rfs.common.OpenSearchClient; @@ -27,7 +27,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import java.io.File; -import java.nio.file.Files; import java.nio.file.Path; /** @@ -35,19 +34,16 @@ */ public class SnapshotStateTest { - private final File SNAPSHOT_DIR = new File("./snapshotRepo"); + @TempDir + private File localDirectory; private ElasticsearchContainer cluster; private ClusterOperations operations; private SimpleRestoreFromSnapshot_ES_7_10 srfs; @BeforeEach public void setUp() throws Exception { - // Make sure the snapshot directory is clean before and after tests run - IOUtils.rm(Path.of(SNAPSHOT_DIR.getAbsolutePath())); - SNAPSHOT_DIR.mkdir(); - // Start the cluster for testing - cluster = new ElasticsearchContainer(ElasticsearchContainer.Version.V7_10_2, SNAPSHOT_DIR.getName()); + cluster = new ElasticsearchContainer(ElasticsearchContainer.Version.V7_10_2); cluster.start(); // Configure operations and rfs implementation @@ -59,7 +55,6 @@ public void setUp() throws Exception { @AfterEach public void tearDown() throws Exception { cluster.close(); - IOUtils.rm(Path.of(SNAPSHOT_DIR.getAbsolutePath())); } @Test @@ -67,14 +62,17 @@ public void SingleSnapshot_SingleDocument() throws Exception { // Setup final var indexName = "my-index"; final var document1Id = "doc1"; - final var document1Body = "{\"foo\":\"bar\"}"; + final var document1Body = "{\"fo$o\":\"bar\"}"; operations.createDocument(indexName, document1Id, document1Body); final var snapshotName = "snapshot-1"; operations.takeSnapshot(snapshotName, indexName); - final var unpackedShardDataDir = Path.of(Files.createTempDirectory("unpacked-shard-data").toFile().getAbsolutePath()); - final var indices = srfs.extraSnapshotIndexData(SNAPSHOT_DIR.getAbsolutePath(), snapshotName, unpackedShardDataDir); + final File snapshotCopy = new File(localDirectory + "/snapshotCopy"); + cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); + + final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data"); + final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); final var client = mock(OpenSearchClient.class); when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty()); @@ -103,8 +101,11 @@ public void SingleSnapshot_SingleDocument_Then_DeletedDocument() throws Exceptio final var snapshotName = "snapshot-delete-item"; operations.takeSnapshot(snapshotName, indexName); - final var unpackedShardDataDir = Path.of(Files.createTempDirectory("unpacked-shard-data").toFile().getAbsolutePath()); - final var indices = srfs.extraSnapshotIndexData(SNAPSHOT_DIR.getAbsolutePath(), snapshotName, unpackedShardDataDir); + final File snapshotCopy = new File(localDirectory + "/snapshotCopy"); + cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); + + final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data"); + final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); final var client = mock(OpenSearchClient.class); when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty()); @@ -135,8 +136,11 @@ public void SingleSnapshot_SingleDocument_Then_UpdateDocument() throws Exception final var snapshotName = "snapshot-delete-item"; operations.takeSnapshot(snapshotName, indexName); - final var unpackedShardDataDir = Path.of(Files.createTempDirectory("unpacked-shard-data").toFile().getAbsolutePath()); - final var indices = srfs.extraSnapshotIndexData(SNAPSHOT_DIR.getAbsolutePath(), snapshotName, unpackedShardDataDir); + final File snapshotCopy = new File(localDirectory + "/snapshotCopy"); + cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); + + final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data"); + final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir); final var client = mock(OpenSearchClient.class); when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty());