Integrated code lifecycle: Structure buildlogs folder (#9304)

src/main/java/de/tum/cit/aet/artemis/assessment/service/ResultService.java

@@ -450,10 +450,11 @@ public Result getResultForParticipationAndCheckAccess(Long participationId, Long
     /**
      * Get a map of result ids to the respective build job ids if build log files for this build job exist.
      *
-     * @param results the results for which to check the availability of build logs
+     * @param results       the results for which to check the availability of build logs
+     * @param participation the participation the results belong to
      * @return a map of result ids to respective build job ids if the build log files exist, null otherwise
      */
-    public Map<Long, String> getLogsAvailabilityForResults(List<Result> results) {
+    public Map<Long, String> getLogsAvailabilityForResults(List<Result> results, Participation participation) {
 
         Map<Long, String> logsAvailability = new HashMap<>();
 
@@ -466,7 +467,7 @@ public Map<Long, String> getLogsAvailabilityForResults(List<Result> results) {
             String buildJobId = resultBuildJobSet.get(resultId);
             if (buildJobId != null) {
 
-                if (buildLogEntryService.buildJobHasLogFile(buildJobId)) {
+                if (buildLogEntryService.buildJobHasLogFile(buildJobId, ((ProgrammingExerciseParticipation) participation).getProgrammingExercise())) {
                     logsAvailability.put(resultId, buildJobId);
                 }
                 else {

src/main/java/de/tum/cit/aet/artemis/assessment/web/ResultResource.java

@@ -187,7 +187,7 @@ public ResponseEntity<Map<Long, String>> getBuildJobIdsForResultsOfParticipation
         Participation participation = participationRepository.findByIdElseThrow(participationId);
         List<Result> results = resultRepository.findAllByParticipationIdOrderByCompletionDateDesc(participationId);
 
-        Map<Long, String> resultBuildJobMap = resultService.getLogsAvailabilityForResults(results);
+        Map<Long, String> resultBuildJobMap = resultService.getLogsAvailabilityForResults(results, participation);
 
         participationAuthCheckService.checkCanAccessParticipationElseThrow(participation);
 

src/main/java/de/tum/cit/aet/artemis/programming/repository/BuildJobRepository.java

@@ -114,4 +114,10 @@ default Page<BuildJob> findAllWithDataByCourseId(long courseId, Pageable pageabl
             """)
     List<BuildJobResultCountDTO> getBuildJobsResultsStatistics(@Param("fromDateTime") ZonedDateTime fromDateTime, @Param("courseId") Long courseId);
 
+    Optional<BuildJob> findByBuildJobId(String buildJobId);
+
+    default BuildJob findByBuildJobIdElseThrow(String buildJobId) {
+        return getValueElseThrow(findByBuildJobId(buildJobId));
+    }
+
 }

src/main/java/de/tum/cit/aet/artemis/programming/repository/ProgrammingExerciseRepository.java

@@ -992,4 +992,14 @@ public String getFetchPath() {
             return fetchPath;
         }
     }
+
+    /**
+     * Find a programming exercise by its id and throw an Exception if it cannot be found
+     *
+     * @param programmingExerciseId of the programming exercise.
+     * @return The programming exercise related to the given id
+     */
+    default ProgrammingExercise findByIdElseThrow(long programmingExerciseId) {
+        return getValueElseThrow(findById(programmingExerciseId));
+    }
 }

src/main/java/de/tum/cit/aet/artemis/programming/service/BuildLogEntryService.java

@@ -24,10 +24,14 @@
 import org.springframework.stereotype.Service;
 
 import de.tum.cit.aet.artemis.core.service.ProfileService;
+import de.tum.cit.aet.artemis.programming.domain.ProgrammingExercise;
 import de.tum.cit.aet.artemis.programming.domain.ProgrammingLanguage;
 import de.tum.cit.aet.artemis.programming.domain.ProgrammingSubmission;
+import de.tum.cit.aet.artemis.programming.domain.build.BuildJob;
 import de.tum.cit.aet.artemis.programming.domain.build.BuildLogEntry;
+import de.tum.cit.aet.artemis.programming.repository.BuildJobRepository;
 import de.tum.cit.aet.artemis.programming.repository.BuildLogEntryRepository;
+import de.tum.cit.aet.artemis.programming.repository.ProgrammingExerciseRepository;
 import de.tum.cit.aet.artemis.programming.repository.ProgrammingSubmissionRepository;
 import de.tum.cit.aet.artemis.programming.service.ci.ContinuousIntegrationService;
 
@@ -43,16 +47,23 @@ public class BuildLogEntryService {
 
     private final ProfileService profileService;
 
+    private final BuildJobRepository buildJobRepository;
+
+    private final ProgrammingExerciseRepository programmingExerciseRepository;
+
     @Value("${artemis.continuous-integration.build-log.file-expiry-days:30}")
     private int expiryDays;
 
     @Value("${artemis.build-logs-path:./build-logs}")
     private Path buildLogsPath;
 
-    public BuildLogEntryService(BuildLogEntryRepository buildLogEntryRepository, ProgrammingSubmissionRepository programmingSubmissionRepository, ProfileService profileService) {
+    public BuildLogEntryService(BuildLogEntryRepository buildLogEntryRepository, ProgrammingSubmissionRepository programmingSubmissionRepository, ProfileService profileService,
+            BuildJobRepository buildJobRepository, ProgrammingExerciseRepository programmingExerciseRepository) {
         this.buildLogEntryRepository = buildLogEntryRepository;
         this.programmingSubmissionRepository = programmingSubmissionRepository;
         this.profileService = profileService;
+        this.buildJobRepository = buildJobRepository;
+        this.programmingExerciseRepository = programmingExerciseRepository;
     }
 
     /**
@@ -282,23 +293,34 @@ public void deleteBuildLogEntriesForProgrammingSubmission(ProgrammingSubmission
     }
 
     /**
-     * Save the build logs for a given submission to a file
+     * Saves a list of build log entries to a file for a specific build job.
+     *
+     * <p>
+     * The log file path is constructed based on the course's short name, the exercise's short name,
+     * and the build job ID. If the directory structure for the logs does not already exist, it is created.
+     * Each log entry is written to the log file in the format of "time\tlog message".
      *
-     * @param buildLogEntries the build logs to save
-     * @param buildJobId      the id of the build job for which to save the build logs
+     * @param buildLogEntries     A list of {@link BuildLogEntry} objects containing the build log information to be saved.
+     * @param buildJobId          The unique identifier of the build job whose logs are being saved.
+     * @param programmingExercise The programming exercise associated with the build job, used to
+     *                                retrieve the course and exercise short names.
+     * @throws IllegalStateException If the directory for storing the logs could not be created.
+     * @throws RuntimeException      If an I/O error occurs while writing the log file.
      */
-    public void saveBuildLogsToFile(List<BuildLogEntry> buildLogEntries, String buildJobId) {
-
-        if (!Files.exists(buildLogsPath)) {
+    public void saveBuildLogsToFile(List<BuildLogEntry> buildLogEntries, String buildJobId, ProgrammingExercise programmingExercise) {
+        String courseShortName = programmingExercise.getCourseViaExerciseGroupOrCourseMember().getShortName();
+        String exerciseShortName = programmingExercise.getShortName();
+        Path exerciseLogsPath = buildLogsPath.resolve(courseShortName).resolve(exerciseShortName);
+        if (!Files.exists(exerciseLogsPath)) {
             try {
-                Files.createDirectories(buildLogsPath);
+                Files.createDirectories(exerciseLogsPath);
             }
             catch (Exception e) {
                 throw new IllegalStateException("Could not create directory for build logs", e);
             }
         }
 
-        Path logPath = buildLogsPath.resolve(buildJobId + ".log");
+        Path logPath = exerciseLogsPath.resolve(buildJobId + ".log");
 
         StringBuilder logsStringBuilder = new StringBuilder();
         for (BuildLogEntry buildLogEntry : buildLogEntries) {
@@ -315,23 +337,49 @@ public void saveBuildLogsToFile(List<BuildLogEntry> buildLogEntries, String buil
     }
 
     /**
-     * Retrieves the build logs for a given submission from a file.
+     * Retrieves the build logs for a specific build job from the file system as a {@link FileSystemResource}.
+     *
+     * <p>
+     * The method first attempts to locate the log file in the directory corresponding to the course
+     * and exercise short names. If the file is not found, it will attempt to retrieve the log from a
+     * parent directory for backward compatibility.
      *
-     * @param buildJobId the id of the build job for which to retrieve the build logs
-     * @return the build logs as a string or null if the file could not be found (e.g. if the build logs have been deleted)
+     * @param buildJobId The unique identifier of the build job whose logs are being retrieved.
+     * @return A {@link FileSystemResource} representing the log file if it exists, or {@code null} if the log file cannot be found.
      */
     public FileSystemResource retrieveBuildLogsFromFileForBuildJob(String buildJobId) {
-        Path logPath = buildLogsPath.resolve(buildJobId + ".log");
+        if (buildJobId.contains("/") || buildJobId.contains("\\") || buildJobId.contains("..")) {
+            log.warn("Invalid build job ID: {}", buildJobId);
+            throw new IllegalArgumentException("Invalid build job ID");
+        }
+
+        ProgrammingExercise programmingExercise = retrieveProgrammingExerciseByBuildJobId(buildJobId);
+        String courseShortName = programmingExercise.getCourseViaExerciseGroupOrCourseMember().getShortName();
+        String exerciseShortName = programmingExercise.getShortName();
+        Path logPath = buildLogsPath.resolve(courseShortName).resolve(exerciseShortName).resolve(buildJobId + ".log");
 
         FileSystemResource fileSystemResource = new FileSystemResource(logPath);
         if (fileSystemResource.exists()) {
             log.debug("Retrieved build logs for build job {} from file {}", buildJobId, logPath);
             return fileSystemResource;
         }
-        else {
-            log.warn("Could not find build logs for build job {} in file {}", buildJobId, logPath);
-            return null;
+
+        // If the file is not found in the exercise directory, try to find it in the parent directory (for backwards compatibility)
+        log.warn("Build log file for build job {} not found at path {}. Searching in Parent directory...", buildJobId, logPath);
+        logPath = buildLogsPath.resolve(buildJobId + ".log");
+        fileSystemResource = new FileSystemResource(logPath);
+        if (fileSystemResource.exists()) {
+            log.debug("Retrieved build logs for build job {} from file {}", buildJobId, logPath);
+            return fileSystemResource;
         }
+
+        log.warn("Could not find build logs for build job {} in file {}", buildJobId, logPath);
+        return null;
+    }
+
+    private ProgrammingExercise retrieveProgrammingExerciseByBuildJobId(String buildJobId) {
+        BuildJob buildJob = buildJobRepository.findByBuildJobIdElseThrow(buildJobId);
+        return programmingExerciseRepository.findByIdElseThrow(buildJob.getExerciseId());
     }
 
     /**
@@ -359,25 +407,86 @@ public void deleteOldBuildLogsFiles() {
         if (!profileService.isSchedulingActive()) {
             return;
         }
+
         log.info("Deleting old build log files");
-        ZonedDateTime now = ZonedDateTime.now();
 
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(buildLogsPath)) {
-            for (Path file : stream) {
-                ZonedDateTime lastModified = ZonedDateTime.ofInstant(Files.getLastModifiedTime(file).toInstant(), now.getZone());
-                if (lastModified.isBefore(now.minusDays(expiryDays))) {
-                    Files.deleteIfExists(file);
-                    log.info("Deleted old build log file {}", file);
-                }
-            }
+        try {
+            deleteExpiredBuildLogFilesRecursively(buildLogsPath);
         }
         catch (IOException e) {
             log.error("Error occurred while trying to delete old build log files", e);
         }
     }
 
-    public boolean buildJobHasLogFile(String buildJobId) {
-        Path logPath = buildLogsPath.resolve(buildJobId + ".log");
+    private void deleteExpiredBuildLogFilesRecursively(Path path) throws IOException {
+        if (!Files.isDirectory(path)) {
+            deleteFileIfExpired(path);
+            return;
+        }
+
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
+            for (Path subPath : stream) {
+                deleteExpiredBuildLogFilesRecursively(subPath);
+            }
+        }
+        catch (IOException e) {
+            log.error("Error occurred while processing directory: {}", path, e);
+        }
+
+        if (!path.equals(buildLogsPath)) {
+            deleteDirectoryIfEmpty(path);
+        }
+    }
+
+    private void deleteFileIfExpired(Path file) throws IOException {
+        ZonedDateTime now = ZonedDateTime.now();
+
+        ZonedDateTime lastModified = ZonedDateTime.ofInstant(Files.getLastModifiedTime(file).toInstant(), now.getZone());
+        if (Files.isRegularFile(file) && lastModified.isBefore(now.minusDays(expiryDays))) {
+            Files.deleteIfExists(file);
+            log.info("Deleted old build log file {}", file);
+        }
+
+    }
+
+    private void deleteDirectoryIfEmpty(Path directory) {
+        if (Files.isDirectory(directory)) {
+            try (DirectoryStream<Path> stream = Files.newDirectoryStream(directory)) {
+                if (!stream.iterator().hasNext()) {
+                    Files.deleteIfExists(directory);
+                    log.info("Deleted empty directory {}", directory);
+                }
+            }
+            catch (IOException e) {
+                log.error("Error occurred while trying to delete empty directory {}", directory, e);
+            }
+        }
+    }
+
+    /**
+     * Checks if the log file for a specific build job exists in the file system.
+     *
+     * <p>
+     * The log file path is constructed based on the course's short name, the exercise's short name,
+     * and the build job ID. The file is expected to be located at:
+     * {@code buildLogsPath/<courseShortName>/<exerciseShortName>/<buildJobId>.log}.
+     *
+     * @param buildJobId          The unique identifier of the build job whose log file is being checked.
+     * @param programmingExercise The programming exercise associated with the build job, used to
+     *                                retrieve the course and exercise short names.
+     * @return {@code true} if the log file exists, otherwise {@code false}.
+     */
+    public boolean buildJobHasLogFile(String buildJobId, ProgrammingExercise programmingExercise) {
+        String courseShortName = programmingExercise.getCourseViaExerciseGroupOrCourseMember().getShortName();
+        String exerciseShortName = programmingExercise.getShortName();
+        Path logPath = buildLogsPath.resolve(courseShortName).resolve(exerciseShortName).resolve(buildJobId + ".log");
+        boolean existsInExerciseFolder = Files.exists(logPath);
+        if (existsInExerciseFolder) {
+            return true;
+        }
+
+        // Check parent folder for backwards compatibility
+        logPath = buildLogsPath.resolve(buildJobId + ".log");
         return Files.exists(logPath);
     }
 

src/main/java/de/tum/cit/aet/artemis/programming/service/localci/LocalCIResultProcessingService.java

@@ -159,6 +159,9 @@ public void processResult() {
 
                 if (participationOptional.isPresent()) {
                     ProgrammingExerciseParticipation participation = (ProgrammingExerciseParticipation) participationOptional.get();
+                    if (participation.getExercise() == null) {
+                        participation.setExercise(programmingExerciseRepository.getProgrammingExerciseFromParticipation(participation));
+                    }
 
                     if (result != null) {
                         programmingMessagingService.notifyUserAboutNewResult(result, participation);
@@ -168,15 +171,15 @@ public void processResult() {
                         programmingMessagingService.notifyUserAboutSubmissionError((Participation) participation,
                                 new BuildTriggerWebsocketError("Result could not be processed", participation.getId()));
                     }
-                }
-            }
 
-            if (!buildLogs.isEmpty()) {
-                if (savedBuildJob != null) {
-                    buildLogEntryService.saveBuildLogsToFile(buildLogs, savedBuildJob.getBuildJobId());
-                }
-                else {
-                    log.warn("Couldn't save build logs as build job {} was not saved", buildJob.id());
+                    if (!buildLogs.isEmpty()) {
+                        if (savedBuildJob != null) {
+                            buildLogEntryService.saveBuildLogsToFile(buildLogs, savedBuildJob.getBuildJobId(), participation.getProgrammingExercise());
+                        }
+                        else {
+                            log.warn("Couldn't save build logs as build job {} was not saved", buildJob.id());
+                        }
+                    }
                 }
             }
 

src/main/java/de/tum/cit/aet/artemis/programming/web/localci/BuildLogResource.java

@@ -43,6 +43,7 @@ public BuildLogResource(BuildLogEntryService buildLogEntryService) {
     public ResponseEntity<Resource> getBuildLogForBuildJob(@PathVariable String buildJobId) {
         log.debug("REST request to get the build log for build job {}", buildJobId);
         HttpHeaders responseHeaders = new HttpHeaders();
+
         FileSystemResource buildLog = buildLogEntryService.retrieveBuildLogsFromFileForBuildJob(buildJobId);
         if (buildLog == null) {
             return new ResponseEntity<>(HttpStatus.NOT_FOUND);

src/test/java/de/tum/cit/aet/artemis/localvcci/LocalCIIntegrationTest.java

@@ -444,7 +444,7 @@ void testBuildLogs() throws IOException {
             assertThat(resultBuildJobSet.iterator().next().buildJobId()).isEqualTo(buildJob.getBuildJobId());
 
             // Assert that the corresponding build job are stored in the file system
-            assertThat(buildLogEntryService.buildJobHasLogFile(buildJob.getBuildJobId())).isTrue();
+            assertThat(buildLogEntryService.buildJobHasLogFile(buildJob.getBuildJobId(), studentParticipation.getProgrammingExercise())).isTrue();
 
             // Retrieve the build logs from the file system
             buildLogs = buildLogEntryService.retrieveBuildLogsFromFileForBuildJob(buildJob.getBuildJobId());