Skip to content

Commit

Permalink
DCopy snapshot data off of container
Browse files Browse the repository at this point in the history 
Signed-off-by: Peter Nied <[email protected]>
  • Loading branch information
@peternied
peternied committed May 28, 2024
1 parent 7a64f93 commit e9a5567
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 21 deletions.
2 changes: 1 addition & 1 deletion RFS/src/test/java/com/rfs/framework/ClusterOperations.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public void createSnapshotRepository() throws IOException {
final var repositoryJson = "{\n" +
" \"type\": \"fs\",\n" +
" \"settings\": {\n" +
" \"location\": \"/snapshots\",\n" +
" \"location\": \"/usr/share/elasticsearch/snapshots\",\n" +
" \"compress\": false\n" +
" }\n" +
"}";
Expand Down
33 changes: 29 additions & 4 deletions RFS/src/test/java/com/rfs/framework/ElasticsearchContainer.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package com.rfs.framework;

import java.io.File;
import java.time.Duration;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.testcontainers.containers.BindMode;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
Expand All @@ -15,21 +15,46 @@
public class ElasticsearchContainer implements AutoCloseable {

private static final Logger logger = LogManager.getLogger(ElasticsearchContainer.class);
private static final String CLUSTER_SNAPSHOT_DIR = "/usr/share/elasticsearch/snapshots";

private final GenericContainer<?> container;
private final Version version;

@SuppressWarnings("resource")
public ElasticsearchContainer(final Version version, final String localSnapshotDirectory) {
public ElasticsearchContainer(final Version version) {
this.version = version;
container = new GenericContainer<>(DockerImageName.parse(this.version.imageName))
.withExposedPorts(9200, 9300)
.withEnv("discovery.type", "single-node")
.withEnv("path.repo", "/snapshots")
.withFileSystemBind(localSnapshotDirectory, "/snapshots", BindMode.READ_WRITE)
.withEnv("path.repo", CLUSTER_SNAPSHOT_DIR)
.waitingFor(Wait.forHttp("/").forPort(9200).forStatusCode(200).withStartupTimeout(Duration.ofMinutes(1)));
}

public void copySnapshotData(final String directory) {
logger.info("Copy stuff was called");
try {
// Execute command to list all files in the directory
final var result = container.execInContainer("sh", "-c", "find " + CLUSTER_SNAPSHOT_DIR + " -type f");
logger.debug("Process Exit Code: " + result.getExitCode());
logger.debug("Standard Output: " + result.getStdout());
logger.debug("Standard Error : " + result.getStderr());
// Process each file and copy it from the container
try (final var lines = result.getStdout().lines()) {
lines.forEach(fullFilePath -> {
final var file = fullFilePath.substring(CLUSTER_SNAPSHOT_DIR.length() + 1);
final var sourcePath = CLUSTER_SNAPSHOT_DIR + "/" + file;
final var destinationPath = directory + "/" + file;
// Make sure the parent directory tree exists before copying
new File(destinationPath).getParentFile().mkdirs();
logger.info("Copying file " + sourcePath + " from container onto " + destinationPath);
container.copyFileFromContainer(sourcePath, destinationPath);
});
}
} catch (final Exception e) {
throw new RuntimeException(e);
}
}

public void start() {
logger.info("Starting ElasticsearchContainer version:" + version.prettyName);
container.start();
Expand Down
36 changes: 20 additions & 16 deletions RFS/src/test/java/com/rfs/integration/SnapshotStateTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package com.rfs.integration;

import org.apache.lucene.util.IOUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.ArgumentCaptor;

import com.rfs.common.OpenSearchClient;
Expand All @@ -27,27 +27,23 @@
import static org.hamcrest.MatcherAssert.assertThat;

import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;

/**
* Tests focused on setting up different snapshot states and then verifying the behavior of RFS towards the target cluster
*/
public class SnapshotStateTest {

private final File SNAPSHOT_DIR = new File("./snapshotRepo");
@TempDir
private File localDirectory;
private ElasticsearchContainer cluster;
private ClusterOperations operations;
private SimpleRestoreFromSnapshot_ES_7_10 srfs;

@BeforeEach
public void setUp() throws Exception {
// Make sure the snapshot directory is clean before and after tests run
IOUtils.rm(Path.of(SNAPSHOT_DIR.getAbsolutePath()));
SNAPSHOT_DIR.mkdir();

// Start the cluster for testing
cluster = new ElasticsearchContainer(ElasticsearchContainer.Version.V7_10_2, SNAPSHOT_DIR.getName());
cluster = new ElasticsearchContainer(ElasticsearchContainer.Version.V7_10_2);
cluster.start();

// Configure operations and rfs implementation
Expand All @@ -59,22 +55,24 @@ public void setUp() throws Exception {
@AfterEach
public void tearDown() throws Exception {
cluster.close();
IOUtils.rm(Path.of(SNAPSHOT_DIR.getAbsolutePath()));
}

@Test
public void SingleSnapshot_SingleDocument() throws Exception {
// Setup
final var indexName = "my-index";
final var document1Id = "doc1";
final var document1Body = "{\"foo\":\"bar\"}";
final var document1Body = "{\"fo$o\":\"bar\"}";
operations.createDocument(indexName, document1Id, document1Body);

final var snapshotName = "snapshot-1";
operations.takeSnapshot(snapshotName, indexName);

final var unpackedShardDataDir = Path.of(Files.createTempDirectory("unpacked-shard-data").toFile().getAbsolutePath());
final var indices = srfs.extraSnapshotIndexData(SNAPSHOT_DIR.getAbsolutePath(), snapshotName, unpackedShardDataDir);
final File snapshotCopy = new File(localDirectory + "/snapshotCopy");
cluster.copySnapshotData(snapshotCopy.getAbsolutePath());

final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data");
final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir);

final var client = mock(OpenSearchClient.class);
when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty());
Expand Down Expand Up @@ -103,8 +101,11 @@ public void SingleSnapshot_SingleDocument_Then_DeletedDocument() throws Exceptio
final var snapshotName = "snapshot-delete-item";
operations.takeSnapshot(snapshotName, indexName);

final var unpackedShardDataDir = Path.of(Files.createTempDirectory("unpacked-shard-data").toFile().getAbsolutePath());
final var indices = srfs.extraSnapshotIndexData(SNAPSHOT_DIR.getAbsolutePath(), snapshotName, unpackedShardDataDir);
final File snapshotCopy = new File(localDirectory + "/snapshotCopy");
cluster.copySnapshotData(snapshotCopy.getAbsolutePath());

final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data");
final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir);

final var client = mock(OpenSearchClient.class);
when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty());
Expand Down Expand Up @@ -135,8 +136,11 @@ public void SingleSnapshot_SingleDocument_Then_UpdateDocument() throws Exception
final var snapshotName = "snapshot-delete-item";
operations.takeSnapshot(snapshotName, indexName);

final var unpackedShardDataDir = Path.of(Files.createTempDirectory("unpacked-shard-data").toFile().getAbsolutePath());
final var indices = srfs.extraSnapshotIndexData(SNAPSHOT_DIR.getAbsolutePath(), snapshotName, unpackedShardDataDir);
final File snapshotCopy = new File(localDirectory + "/snapshotCopy");
cluster.copySnapshotData(snapshotCopy.getAbsolutePath());

final var unpackedShardDataDir = Path.of(localDirectory.getAbsolutePath() + "/unpacked-shard-data");
final var indices = srfs.extraSnapshotIndexData(snapshotCopy.getAbsolutePath(), snapshotName, unpackedShardDataDir);

final var client = mock(OpenSearchClient.class);
when(client.sendBulkRequest(any(), any())).thenReturn(Mono.empty());
Expand Down

0 comments on commit e9a5567

Please sign in to comment.