diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index c757db36e..d4f56261d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -27,8 +27,10 @@ import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; import java.io.File; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Set; +import java.util.stream.Stream; import javax.inject.Inject; import javax.inject.Singleton; import org.apache.commons.io.FileUtils; @@ -138,4 +140,19 @@ private Void deleteIfEmpty(File dir) { if (FileUtils.sizeOfDirectory(dir) == 0) FileUtils.deleteQuietly(dir); return null; } + + public static int countFilesInBackupDir(IConfiguration config) throws Exception { + int totalFileCount = 0; + Set backupDirectories = + AbstractBackup.getBackupDirectories(config, INCREMENTAL_BACKUP_FOLDER); + for (Path backupDir : backupDirectories) { + try (Stream stream = Files.list(backupDir)) { + totalFileCount += stream.filter(Files::isRegularFile).count(); + } catch (Exception e) { + logger.error("Failed to get files in backups directory. {}", e.getMessage()); + e.printStackTrace(); + } + } + return totalFileCount; + } } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java index 846574d8a..405dea4ef 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java @@ -25,6 +25,8 @@ import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.tuner.CassandraTunerService; +import java.util.HashMap; +import java.util.Map; import javax.inject.Inject; import org.apache.commons.lang3.math.Fraction; @@ -101,4 +103,11 @@ public void updateServicePre() throws Exception { @Override public void updateServicePost() throws Exception {} + + public Map countPendingBackupFiles() throws Exception { + Map backupFiles = new HashMap(); + backupFiles.put("snapshotFiles", snapshotMetaTask.countFilesInSnapshotDir(configuration)); + backupFiles.put("incrementalFiles", IncrementalBackup.countFilesInBackupDir(configuration)); + return backupFiles; + } } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index 10afa1cec..394e8056d 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -48,6 +48,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Stream; import javax.inject.Inject; import javax.inject.Named; import javax.inject.Singleton; @@ -442,4 +443,26 @@ public void onFailure(Throwable t) { }; Futures.addCallback(future, callback, MoreExecutors.directExecutor()); } + + public int countFilesInSnapshotDir(IConfiguration config) throws Exception { + int totalFileCount = 0; + Set snapshotDirectories = + AbstractBackup.getBackupDirectories(config, SNAPSHOT_FOLDER); + for (Path snapshotDir : snapshotDirectories) { + try (DirectoryStream directoryStream = + Files.newDirectoryStream(snapshotDir, Files::isDirectory)) { + for (Path backupDir : directoryStream) { + if (backupDir.toFile().getName().startsWith(SNAPSHOT_PREFIX)) { + try (Stream stream = Files.list(backupDir)) { + totalFileCount += stream.filter(Files::isRegularFile).count(); + } + } + } + } catch (Exception e) { + logger.error("Failed to get files in snapshot directory. {}", e.getMessage()); + e.printStackTrace(); + } + } + return totalFileCount; + } } diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java index 5813b7599..8def18a5e 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java @@ -17,19 +17,24 @@ package com.netflix.priam.resources; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.priam.PriamServer; import com.netflix.priam.backup.*; import com.netflix.priam.backupv2.BackupTTLTask; import com.netflix.priam.backupv2.BackupV2Service; import com.netflix.priam.backupv2.IMetaProxy; import com.netflix.priam.backupv2.SnapshotMetaTask; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.DateUtil.DateRange; import com.netflix.priam.utils.GsonJsonSerializer; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import javax.inject.Inject; @@ -55,6 +60,9 @@ public class BackupServletV2 { private final Provider pathProvider; private final BackupV2Service backupService; private final BackupNotificationMgr backupNotificationMgr; + private final PriamServer priamServer; + + private final BackupMetrics backupMetrics; private static final String REST_SUCCESS = "[\"ok\"]"; @Inject @@ -68,7 +76,9 @@ public BackupServletV2( @Named("v2") IMetaProxy metaV2Proxy, Provider pathProvider, BackupV2Service backupService, - BackupNotificationMgr backupNotificationMgr) { + BackupNotificationMgr backupNotificationMgr, + PriamServer priamServer, + BackupMetrics backupMetrics) { this.backupStatusMgr = backupStatusMgr; this.backupVerification = backupVerification; this.snapshotMetaService = snapshotMetaService; @@ -78,6 +88,8 @@ public BackupServletV2( this.pathProvider = pathProvider; this.backupService = backupService; this.backupNotificationMgr = backupNotificationMgr; + this.priamServer = priamServer; + this.backupMetrics = backupMetrics; } @GET @@ -181,4 +193,26 @@ public Response list(@PathParam("daterange") String daterange) throws Exception .collect(Collectors.toList()))) .build(); } + + @GET + @Path("/state/{hours}") + public Response backupState(@PathParam("hours") int hours) throws Exception { + Map responseMap = new HashMap<>(); + + responseMap.put("tasksQueued", fs.getUploadTasksQueued()); + responseMap.put("queueSize", priamServer.getConfiguration().getBackupQueueSize()); + for (Map.Entry entry : + backupService.countPendingBackupFiles().entrySet()) { + responseMap.put(entry.getKey(), entry.getValue()); + } + + List latestBackupMetadata = + backupStatusMgr.getLatestBackupMetadata( + new DateRange(Instant.now().minus(hours, ChronoUnit.HOURS), Instant.now())); + responseMap.put("latestBackupMetadata", latestBackupMetadata); + + ObjectMapper mapper = new ObjectMapper(); + String jsonResponse = mapper.writeValueAsString(responseMap); + return Response.ok(jsonResponse).build(); + } }