Skip to content

Commit

Permalink
Return missing files from snapshot verification continuing to margina…
Browse files Browse the repository at this point in the history 
…lize BackupVerificationResult. Return all results valid or otherwise.
  • Loading branch information
@mattl-netflix
mattl-netflix committed Apr 4, 2023
1 parent 373069a commit 6f1a435
Show file tree
Hide file tree
Showing 7 changed files with 151 additions and 178 deletions.
42 changes: 13 additions & 29 deletions priam/src/main/java/com/netflix/priam/backup/BackupVerification.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
*/
package com.netflix.priam.backup;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.netflix.priam.backupv2.IMetaProxy;
import com.netflix.priam.utils.DateUtil;
import com.netflix.priam.utils.DateUtil.DateRange;
Expand Down Expand Up @@ -68,46 +70,29 @@ public IMetaProxy getMetaProxy(BackupVersion backupVersion) {
return null;
}

public Optional<BackupVerificationResult> verifyLatestBackup(
public ImmutableMap<BackupMetadata, ImmutableSet<String>> findMissingBackupFilesInRange(
BackupVersion backupVersion, boolean force, DateRange dateRange)
throws IllegalArgumentException {
IMetaProxy metaProxy = getMetaProxy(backupVersion);
ImmutableMap.Builder<BackupMetadata, ImmutableSet<String>> mapBuilder =
ImmutableMap.builder();
for (BackupMetadata backupMetadata :
backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange)) {
if (backupMetadata.getLastValidated() == null || force) {
Optional<BackupVerificationResult> result = verifyBackup(metaProxy, backupMetadata);
if (result.isPresent()) {
return result;
}
} else {
updateLatestResult(backupMetadata);
return Optional.of(latestResult);
}
List<String> missingFiles =
backupMetadata.getLastValidated() == null || force
? verifyBackup(metaProxy, backupMetadata).filesInMetaOnly
: new ArrayList<>();
mapBuilder.put(backupMetadata, ImmutableSet.copyOf(missingFiles));
}
latestResult = null;
return Optional.empty();
}

public List<BackupMetadata> verifyBackupsInRange(
BackupVersion backupVersion, DateRange dateRange) throws IllegalArgumentException {
IMetaProxy metaProxy = getMetaProxy(backupVersion);
List<BackupMetadata> results = new ArrayList<>();
for (BackupMetadata backupMetadata :
backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange)) {
if (backupMetadata.getLastValidated() != null
|| verifyBackup(metaProxy, backupMetadata).isPresent()) {
results.add(backupMetadata);
}
}
return results;
return mapBuilder.build();
}

/** returns the latest valid backup verification result if we have found one within the SLO * */
public Optional<Instant> getLatestVerfifiedBackupTime() {
return latestResult == null ? Optional.empty() : Optional.of(latestResult.snapshotInstant);
}

private Optional<BackupVerificationResult> verifyBackup(
private BackupVerificationResult verifyBackup(
IMetaProxy metaProxy, BackupMetadata latestBackupMetaData) {
Path metadataLocation = Paths.get(latestBackupMetaData.getSnapshotLocation());
metadataLocation = metadataLocation.subpath(1, metadataLocation.getNameCount());
Expand All @@ -119,9 +104,8 @@ private Optional<BackupVerificationResult> verifyBackup(
Date now = new Date(DateUtil.getInstant().toEpochMilli());
latestBackupMetaData.setLastValidated(now);
backupStatusMgr.update(latestBackupMetaData);
return Optional.of(result);
}
return Optional.empty();
return result;
}

private void updateLatestResult(BackupMetadata backupMetadata) {
Expand Down
25 changes: 17 additions & 8 deletions priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@
import com.netflix.priam.utils.DateUtil.DateRange;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.slf4j.Logger;
Expand Down Expand Up @@ -84,21 +86,28 @@ public void execute() throws Exception {
Instant slo =
now.minus(backupRestoreConfig.getBackupVerificationSLOInHours(), ChronoUnit.HOURS);
DateRange dateRange = new DateRange(slo, now);
List<BackupMetadata> verifiedBackups =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_META_SERVICE, dateRange);
Set<BackupMetadata> verifiedBackups =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE, false /* force */, dateRange)
.entrySet()
.stream()
.filter(entry -> entry.getValue().isEmpty())
.map(Map.Entry::getKey)
.collect(Collectors.toSet());

verifiedBackups
.stream()
.filter(result -> result.getLastValidated().toInstant().isAfter(now))
.filter(metadata -> metadata.getLastValidated().toInstant().isAfter(now))
.forEach(
result -> {
metadata -> {
logger.info(
"Sending {} message for backup: {}",
AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED,
result.getSnapshotLocation());
metadata.getSnapshotLocation());
backupNotificationMgr.notify(
result.getSnapshotLocation(), result.getStart().toInstant());
metadata.getSnapshotLocation(),
metadata.getStart().toInstant());
});

if (verifiedBackups.isEmpty()) {
Expand Down
18 changes: 7 additions & 11 deletions priam/src/main/java/com/netflix/priam/resources/BackupServlet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,16 @@
*/
package com.netflix.priam.resources;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.netflix.priam.backup.*;
import com.netflix.priam.backup.AbstractBackupPath.BackupFileType;
import com.netflix.priam.config.IBackupRestoreConfig;
import com.netflix.priam.config.IConfiguration;
import com.netflix.priam.scheduler.PriamScheduler;
import com.netflix.priam.utils.DateUtil;
import com.netflix.priam.utils.DateUtil.DateRange;
import com.netflix.priam.utils.GsonJsonSerializer;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.*;
Expand Down Expand Up @@ -215,19 +218,12 @@ public Response snapshotsByDate(@PathParam("date") String date) throws Exception
@Produces(MediaType.APPLICATION_JSON)
public Response validateSnapshotByDate(
@PathParam("daterange") String daterange,
@DefaultValue("false") @QueryParam("force") boolean force)
throws Exception {
@DefaultValue("false") @QueryParam("force") boolean force) {
DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange);
Optional<BackupVerificationResult> result =
backupVerification.verifyLatestBackup(
ImmutableMap<BackupMetadata, ImmutableSet<String>> result =
backupVerification.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_BACKUP, force, dateRange);
if (!result.isPresent()) {
return Response.noContent()
.entity("No valid meta found for provided time range")
.build();
}

return Response.ok(result.get().toString()).build();
return Response.ok(GsonJsonSerializer.getGson().toJson(result)).build();
}

/*
Expand Down
17 changes: 6 additions & 11 deletions priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package com.netflix.priam.resources;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.netflix.priam.backup.*;
import com.netflix.priam.backupv2.BackupTTLTask;
import com.netflix.priam.backupv2.BackupV2Service;
Expand Down Expand Up @@ -121,19 +123,12 @@ public Response info(@PathParam("date") String date) {
@Path("/validate/{daterange}")
public Response validateV2SnapshotByDate(
@PathParam("daterange") String daterange,
@DefaultValue("false") @QueryParam("force") boolean force)
throws Exception {
@DefaultValue("false") @QueryParam("force") boolean force) {
DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange);
Optional<BackupVerificationResult> result =
backupVerification.verifyLatestBackup(
ImmutableMap<BackupMetadata, ImmutableSet<String>> result =
backupVerification.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE, force, dateRange);
if (!result.isPresent()) {
return Response.noContent()
.entity("No valid meta found for provided time range")
.build();
}

return Response.ok(result.get().toString()).build();
return Response.ok(GsonJsonSerializer.getGson().toJson(result)).build();
}

@GET
Expand Down
112 changes: 57 additions & 55 deletions priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@

package com.netflix.priam.backup;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.truth.Truth;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.netflix.priam.backup.AbstractBackupPath.BackupFileType;
Expand All @@ -31,9 +34,7 @@
import java.nio.file.Paths;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.*;
import mockit.Mock;
import mockit.MockUp;
import org.apache.commons.io.FileUtils;
Expand Down Expand Up @@ -91,34 +92,43 @@ public void cleanup() {
}

@Test
public void noBackup() throws Exception {
Optional<BackupVerificationResult> backupVerificationResultOptinal =
backupVerification.verifyLatestBackup(
public void noBackup() {
ImmutableMap<BackupMetadata, ImmutableSet<String>> missingFiles =
backupVerification.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_BACKUP,
false,
new DateRange(Instant.now(), Instant.now()));
Assert.assertFalse(backupVerificationResultOptinal.isPresent());
Truth.assertThat(missingFiles).isEmpty();

backupVerificationResultOptinal =
backupVerification.verifyLatestBackup(
missingFiles =
backupVerification.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
false,
new DateRange(Instant.now(), Instant.now()));
Assert.assertFalse(backupVerificationResultOptinal.isPresent());
Truth.assertThat(missingFiles).isEmpty();
}

@Test
public void noBackupDateRange() throws Exception {
List<BackupMetadata> backupVerificationResults =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_BACKUP, new DateRange(Instant.now(), Instant.now()));
Assert.assertFalse(backupVerificationResults.size() > 0);
long foundBackups =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_BACKUP,
false /* force */,
new DateRange(Instant.now(), Instant.now()))
.entrySet()
.size();
Truth.assertThat(foundBackups).isEqualTo(0L);

backupVerificationResults =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
new DateRange(Instant.now(), Instant.now()));
Assert.assertFalse(backupVerificationResults.size() > 0);
foundBackups =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
false /* force */,
new DateRange(Instant.now(), Instant.now()))
.entrySet()
.size();
Truth.assertThat(foundBackups).isEqualTo(0L);
}

private void setUp() throws Exception {
Expand Down Expand Up @@ -152,13 +162,12 @@ private void setUp() throws Exception {
public void verifyBackupVersion1() throws Exception {
setUp();
// Verify for backup version 1.0
Optional<BackupVerificationResult> backupVerificationResultOptinal =
backupVerification.verifyLatestBackup(
ImmutableMap<BackupMetadata, ImmutableSet<String>> missingFiles =
backupVerification.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_BACKUP,
false,
new DateRange(backupDate + "," + backupDate));
Assert.assertTrue(backupVerificationResultOptinal.isPresent());
Assert.assertEquals(Instant.EPOCH, backupVerificationResultOptinal.get().snapshotInstant);
Truth.assertThat(missingFiles.values().stream().allMatch(Set::isEmpty)).isTrue();
Optional<BackupMetadata> backupMetadata =
backupStatusMgr
.getLatestBackupMetadata(
Expand All @@ -184,12 +193,17 @@ public void verifyBackupVersion1() throws Exception {
public void verifyBackupVersion1DateRange() throws Exception {
setUp();
// Verify for backup version 1.0
List<BackupMetadata> backupVerificationResults =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_BACKUP,
new DateRange(backupDate + "," + backupDateEnd));
Assert.assertTrue(!backupVerificationResults.isEmpty());
Assert.assertTrue(backupVerificationResults.size() == numFakeBackups);
long missingFilesCount =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_BACKUP,
false /* force */,
new DateRange(backupDate + "," + backupDateEnd))
.values()
.stream()
.filter(AbstractCollection::isEmpty)
.count();
Truth.assertThat(missingFilesCount).isEqualTo(numFakeBackups);
List<BackupMetadata> backupMetadata =
backupStatusMgr.getLatestBackupMetadata(
BackupVersion.SNAPSHOT_BACKUP,
Expand All @@ -211,14 +225,12 @@ public void verifyBackupVersion1DateRange() throws Exception {
public void verifyBackupVersion2() throws Exception {
setUp();
// Verify for backup version 2.0
Optional<BackupVerificationResult> backupVerificationResultOptinal =
backupVerification.verifyLatestBackup(
ImmutableMap<BackupMetadata, ImmutableSet<String>> missingFiles =
backupVerification.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
false,
new DateRange(backupDate + "," + backupDate));
Assert.assertTrue(backupVerificationResultOptinal.isPresent());
Assert.assertEquals(Instant.EPOCH, backupVerificationResultOptinal.get().snapshotInstant);
Assert.assertEquals("some_random", backupVerificationResultOptinal.get().remotePath);
Truth.assertThat(missingFiles.values().stream().allMatch(Set::isEmpty)).isTrue();

Optional<BackupMetadata> backupMetadata =
backupStatusMgr
Expand All @@ -230,21 +242,6 @@ public void verifyBackupVersion2() throws Exception {
Assert.assertTrue(backupMetadata.isPresent());
Assert.assertNotNull(backupMetadata.get().getLastValidated());

// Retry the verification, it should not try and re-verify
backupVerificationResultOptinal =
backupVerification.verifyLatestBackup(
BackupVersion.SNAPSHOT_META_SERVICE,
false,
new DateRange(backupDate + "," + backupDate));
Assert.assertTrue(backupVerificationResultOptinal.isPresent());
Assert.assertEquals(
DateUtil.parseInstant(backupDate),
backupVerificationResultOptinal.get().snapshotInstant);
Assert.assertNotEquals("some_random", backupVerificationResultOptinal.get().remotePath);
Assert.assertEquals(
location.subpath(1, location.getNameCount()).toString(),
backupVerificationResultOptinal.get().remotePath);

backupMetadata =
backupStatusMgr
.getLatestBackupMetadata(
Expand All @@ -260,12 +257,17 @@ public void verifyBackupVersion2() throws Exception {
public void verifyBackupVersion2DateRange() throws Exception {
setUp();
// Verify for backup version 2.0
List<BackupMetadata> backupVerificationResults =
backupVerification.verifyBackupsInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
new DateRange(backupDate + "," + backupDateEnd));
Assert.assertTrue(!backupVerificationResults.isEmpty());
Assert.assertTrue(backupVerificationResults.size() == numFakeBackups);
long missingFilesCount =
backupVerification
.findMissingBackupFilesInRange(
BackupVersion.SNAPSHOT_META_SERVICE,
false /* force */,
new DateRange(backupDate + "," + backupDateEnd))
.values()
.stream()
.filter(AbstractCollection::isEmpty)
.count();
Truth.assertThat(missingFilesCount).isEqualTo(numFakeBackups);
List<BackupMetadata> backupMetadata =
backupStatusMgr.getLatestBackupMetadata(
BackupVersion.SNAPSHOT_META_SERVICE,
Expand Down
Loading

0 comments on commit 6f1a435

Please sign in to comment.