endpoint to return backup queue, metatdata and pending files count

Netflix · Jun 20, 2024 · 10840be · 10840be
1 parent fee4ddf
commit 10840be
Show file tree

Hide file tree

Showing 4 changed files with 84 additions and 1 deletion.
diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java
@@ -27,8 +27,10 @@
 import com.netflix.priam.scheduler.SimpleTimer;
 import com.netflix.priam.scheduler.TaskTimer;
 import java.io.File;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Set;
+import java.util.stream.Stream;
 import javax.inject.Inject;
 import javax.inject.Singleton;
 import org.apache.commons.io.FileUtils;
@@ -138,4 +140,19 @@ private Void deleteIfEmpty(File dir) {
         if (FileUtils.sizeOfDirectory(dir) == 0) FileUtils.deleteQuietly(dir);
         return null;
     }
+
+    public static int countFilesInBackupDir(IConfiguration config) throws Exception {
+        int totalFileCount = 0;
+        Set<Path> backupDirectories =
+                AbstractBackup.getBackupDirectories(config, INCREMENTAL_BACKUP_FOLDER);
+        for (Path backupDir : backupDirectories) {
+            try (Stream<Path> stream = Files.list(backupDir)) {
+                totalFileCount += stream.filter(Files::isRegularFile).count();
+            } catch (Exception e) {
+                logger.error("Failed to get files in backups directory. {}", e.getMessage());
+                e.printStackTrace();
+            }
+        }
+        return totalFileCount;
+    }
 }
diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java
@@ -25,6 +25,8 @@
 import com.netflix.priam.scheduler.PriamScheduler;
 import com.netflix.priam.scheduler.TaskTimer;
 import com.netflix.priam.tuner.CassandraTunerService;
+import java.util.HashMap;
+import java.util.Map;
 import javax.inject.Inject;
 import org.apache.commons.lang3.math.Fraction;
 
@@ -101,4 +103,11 @@ public void updateServicePre() throws Exception {
 
     @Override
     public void updateServicePost() throws Exception {}
+
+    public Map<String, Integer> countPendingBackupFiles() throws Exception {
+        Map<String, Integer> backupFiles = new HashMap<String, Integer>();
+        backupFiles.put("snapshotFiles", snapshotMetaTask.countFilesInSnapshotDir(configuration));
+        backupFiles.put("incrementalFiles", IncrementalBackup.countFilesInBackupDir(configuration));
+        return backupFiles;
+    }
 }
diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java
@@ -48,6 +48,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Stream;
 import javax.inject.Inject;
 import javax.inject.Named;
 import javax.inject.Singleton;
@@ -442,4 +443,26 @@ public void onFailure(Throwable t) {
                 };
         Futures.addCallback(future, callback, MoreExecutors.directExecutor());
     }
+
+    public int countFilesInSnapshotDir(IConfiguration config) throws Exception {
+        int totalFileCount = 0;
+        Set<Path> snapshotDirectories =
+                AbstractBackup.getBackupDirectories(config, SNAPSHOT_FOLDER);
+        for (Path snapshotDir : snapshotDirectories) {
+            try (DirectoryStream<Path> directoryStream =
+                    Files.newDirectoryStream(snapshotDir, Files::isDirectory)) {
+                for (Path backupDir : directoryStream) {
+                    if (backupDir.toFile().getName().startsWith(SNAPSHOT_PREFIX)) {
+                        try (Stream<Path> stream = Files.list(backupDir)) {
+                            totalFileCount += stream.filter(Files::isRegularFile).count();
+                        }
+                    }
+                }
+            } catch (Exception e) {
+                logger.error("Failed to get files in snapshot directory. {}", e.getMessage());
+                e.printStackTrace();
+            }
+        }
+        return totalFileCount;
+    }
 }
diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java
@@ -17,19 +17,24 @@
 
 package com.netflix.priam.resources;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.netflix.priam.PriamServer;
 import com.netflix.priam.backup.*;
 import com.netflix.priam.backupv2.BackupTTLTask;
 import com.netflix.priam.backupv2.BackupV2Service;
 import com.netflix.priam.backupv2.IMetaProxy;
 import com.netflix.priam.backupv2.SnapshotMetaTask;
 import com.netflix.priam.config.IConfiguration;
+import com.netflix.priam.merics.BackupMetrics;
 import com.netflix.priam.notification.BackupNotificationMgr;
 import com.netflix.priam.utils.DateUtil;
 import com.netflix.priam.utils.DateUtil.DateRange;
 import com.netflix.priam.utils.GsonJsonSerializer;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
 import javax.inject.Inject;
@@ -55,6 +60,9 @@ public class BackupServletV2 {
     private final Provider<AbstractBackupPath> pathProvider;
     private final BackupV2Service backupService;
     private final BackupNotificationMgr backupNotificationMgr;
+    private final PriamServer priamServer;
+
+    private final BackupMetrics backupMetrics;
     private static final String REST_SUCCESS = "[\"ok\"]";
 
     @Inject
@@ -68,7 +76,9 @@ public BackupServletV2(
             @Named("v2") IMetaProxy metaV2Proxy,
             Provider<AbstractBackupPath> pathProvider,
             BackupV2Service backupService,
-            BackupNotificationMgr backupNotificationMgr) {
+            BackupNotificationMgr backupNotificationMgr,
+            PriamServer priamServer,
+            BackupMetrics backupMetrics) {
         this.backupStatusMgr = backupStatusMgr;
         this.backupVerification = backupVerification;
         this.snapshotMetaService = snapshotMetaService;
@@ -78,6 +88,8 @@ public BackupServletV2(
         this.pathProvider = pathProvider;
         this.backupService = backupService;
         this.backupNotificationMgr = backupNotificationMgr;
+        this.priamServer = priamServer;
+        this.backupMetrics = backupMetrics;
     }
 
     @GET
@@ -181,4 +193,26 @@ public Response list(@PathParam("daterange") String daterange) throws Exception
                                                 .collect(Collectors.toList())))
                 .build();
     }
+
+    @GET
+    @Path("/state/{hours}")
+    public Response backupState(@PathParam("hours") int hours) throws Exception {
+        Map<String, Object> responseMap = new HashMap<>();
+
+        responseMap.put("tasksQueued", fs.getUploadTasksQueued());
+        responseMap.put("queueSize", priamServer.getConfiguration().getBackupQueueSize());
+        for (Map.Entry<String, Integer> entry :
+                backupService.countPendingBackupFiles().entrySet()) {
+            responseMap.put(entry.getKey(), entry.getValue());
+        }
+
+        List<BackupMetadata> latestBackupMetadata =
+                backupStatusMgr.getLatestBackupMetadata(
+                        new DateRange(Instant.now().minus(hours, ChronoUnit.HOURS), Instant.now()));
+        responseMap.put("latestBackupMetadata", latestBackupMetadata);
+
+        ObjectMapper mapper = new ObjectMapper();
+        String jsonResponse = mapper.writeValueAsString(responseMap);
+        return Response.ok(jsonResponse).build();
+    }
 }