Development: Cache active users in metrics calculation to improve per…

…formance (#7480)

src/main/java/de/tum/in/www1/artemis/config/MetricsBean.java

@@ -33,7 +33,6 @@
 import de.tum.in.www1.artemis.domain.enumeration.ExerciseType;
 import de.tum.in.www1.artemis.domain.exam.Exam;
 import de.tum.in.www1.artemis.domain.metrics.ExerciseTypeMetricsEntry;
-import de.tum.in.www1.artemis.domain.statistics.StatisticsEntry;
 import de.tum.in.www1.artemis.repository.CourseRepository;
 import de.tum.in.www1.artemis.repository.ExamRepository;
 import de.tum.in.www1.artemis.repository.ExerciseRepository;
@@ -89,6 +88,12 @@ public class MetricsBean {
 
     private final StatisticsRepository statisticsRepository;
 
+    /**
+     * List that stores active usernames (users with a submission within the last 14 days) which is refreshed
+     * every 60 minutes.
+     */
+    private List<String> cachedActiveUserNames;
+
     // Public metrics
     private final AtomicInteger activeCoursesGauge = new AtomicInteger(0);
 
@@ -156,6 +161,9 @@ public MetricsBean(MeterRegistry meterRegistry, Environment env, TaskScheduler t
             // Should only be activated if the scheduling profile is present, because these metrics are the same for all instances
             this.scheduledMetricsEnabled = true;
 
+            // Initial calculation is done in constructor to ensure the values are present before the first metrics are calculated
+            calculateCachedActiveUserNames();
+
             registerExerciseAndExamMetrics();
             registerPublicArtemisMetrics();
         }
@@ -277,6 +285,24 @@ private void registerStudentExamMetrics() {
                 .description("Number of exams starting within the next minutes multiplied with students in the course").register(meterRegistry);
     }
 
+    /**
+     * Calculate active users (active within the last 14 days) and store them in a List.
+     * The calculation is performed every 60 minutes.
+     * The initial calculation is done in the constructor to ensure it is done BEFORE {@link #recalculateMetrics()}
+     * is called.
+     */
+    @Scheduled(fixedRate = 60 * 60 * 1000, initialDelay = 60 * 60 * 1000) // Every 60 minutes
+    public void calculateCachedActiveUserNames() {
+        var startDate = System.currentTimeMillis();
+
+        // The authorization object has to be set because this method is not called by a user but by the scheduler
+        SecurityUtils.setAuthorizationObject();
+
+        cachedActiveUserNames = statisticsRepository.getActiveUserNames(ZonedDateTime.now().minusDays(14), ZonedDateTime.now());
+
+        log.info("calculateCachedActiveUserLogins took {}ms", System.currentTimeMillis() - startDate);
+    }
+
     /**
      * Update exams & exercise metrics.
      * The update (and recalculation) is performed every 5 minutes.
@@ -292,22 +318,21 @@ public void recalculateMetrics() {
         // The authorization object has to be set because this method is not called by a user but by the scheduler
         SecurityUtils.setAuthorizationObject();
 
-        var activeUsers = statisticsRepository.getActiveUsers(ZonedDateTime.now().minusDays(14), ZonedDateTime.now());
-        var activeUserNames = activeUsers.stream().map(StatisticsEntry::getUsername).toList();
+        // The active users are cached and updated every 60 minutes to reduce the database load
 
         // Exercise metrics
-        updateMultiGaugeMetricsEntryForMinuteRanges(dueExerciseGauge, activeUserNames,
+        updateMultiGaugeMetricsEntryForMinuteRanges(dueExerciseGauge, cachedActiveUserNames,
                 (now, endDate, activeUserNamesUnused) -> exerciseRepository.countExercisesWithEndDateBetweenGroupByExerciseType(now, endDate));
-        updateMultiGaugeMetricsEntryForMinuteRanges(dueExerciseStudentMultiplierGauge, activeUserNames,
+        updateMultiGaugeMetricsEntryForMinuteRanges(dueExerciseStudentMultiplierGauge, cachedActiveUserNames,
                 (now, endDate, activeUserNamesUnused) -> exerciseRepository.countStudentsInExercisesWithDueDateBetweenGroupByExerciseType(now, endDate));
-        updateMultiGaugeMetricsEntryForMinuteRanges(dueExerciseStudentMultiplierActive14DaysGauge, activeUserNames,
+        updateMultiGaugeMetricsEntryForMinuteRanges(dueExerciseStudentMultiplierActive14DaysGauge, cachedActiveUserNames,
                 exerciseRepository::countActiveStudentsInExercisesWithDueDateBetweenGroupByExerciseType);
 
-        updateMultiGaugeMetricsEntryForMinuteRanges(releaseExerciseGauge, activeUserNames,
+        updateMultiGaugeMetricsEntryForMinuteRanges(releaseExerciseGauge, cachedActiveUserNames,
                 (now, endDate, activeUserNamesUnused) -> exerciseRepository.countExercisesWithReleaseDateBetweenGroupByExerciseType(now, endDate));
-        updateMultiGaugeMetricsEntryForMinuteRanges(releaseExerciseStudentMultiplierGauge, activeUserNames,
+        updateMultiGaugeMetricsEntryForMinuteRanges(releaseExerciseStudentMultiplierGauge, cachedActiveUserNames,
                 (now, endDate, activeUserNamesUnused) -> exerciseRepository.countStudentsInExercisesWithReleaseDateBetweenGroupByExerciseType(now, endDate));
-        updateMultiGaugeMetricsEntryForMinuteRanges(releaseExerciseStudentMultiplierActive14DaysGauge, activeUserNames,
+        updateMultiGaugeMetricsEntryForMinuteRanges(releaseExerciseStudentMultiplierActive14DaysGauge, cachedActiveUserNames,
                 exerciseRepository::countActiveStudentsInExercisesWithReleaseDateBetweenGroupByExerciseType);
 
         // Exam metrics

src/main/java/de/tum/in/www1/artemis/repository/StatisticsRepository.java

@@ -81,6 +81,18 @@ List<StatisticsEntry> getTotalSubmissionsForExercise(@Param("startDate") ZonedDa
             """)
     List<StatisticsEntry> getActiveUsers(@Param("startDate") ZonedDateTime startDate, @Param("endDate") ZonedDateTime endDate);
 
+    @Query("""
+            SELECT DISTINCT u.login
+            FROM User u, Submission s, StudentParticipation p
+            WHERE
+                s.participation.id = p.id AND
+                p.student.id = u.id AND
+                s.submissionDate >= :startDate AND
+                s.submissionDate <= :endDate AND
+                u.login NOT LIKE '%test%'
+            """)
+    List<String> getActiveUserNames(@Param("startDate") ZonedDateTime startDate, @Param("endDate") ZonedDateTime endDate);
+
     /**
      * Count users that were active within the given date range.
      * Users are considered as active if they created a submission within the given date range

src/test/java/de/tum/in/www1/artemis/config/MetricsBeanTest.java

@@ -351,6 +351,8 @@ void testPrometheusMetricsExercises() {
         quizExerciseUtilService.saveQuizSubmission(exerciseUtilService.getFirstExerciseWithType(course1, QuizExercise.class), ParticipationFactory.generateQuizSubmission(true),
                 users.get(0).getLogin());
 
+        // We have to first refresh the active users and then the metrics to ensure the data is updated correctly
+        metricsBean.calculateCachedActiveUserNames();
         metricsBean.recalculateMetrics();
 
         // Should now have one active user
@@ -364,6 +366,8 @@ void testPrometheusMetricsExercises() {
         quizExerciseUtilService.saveQuizSubmission(exerciseUtilService.getFirstExerciseWithType(course1, QuizExercise.class), ParticipationFactory.generateQuizSubmission(true),
                 users.get(1).getLogin());
 
+        // We have to first refresh the active users and then the metrics to ensure the data is updated correctly
+        metricsBean.calculateCachedActiveUserNames();
         metricsBean.recalculateMetrics();
 
         // Should now have two active users