Text exercises: Add preliminary AI feedback requests for students on …

…text exercises using Athena (#9241)

src/main/java/de/tum/in/www1/artemis/domain/Exercise.java

@@ -49,6 +49,7 @@
 import com.fasterxml.jackson.annotation.JsonView;
 
 import de.tum.in.www1.artemis.domain.competency.CourseCompetency;
+import de.tum.in.www1.artemis.domain.enumeration.AssessmentType;
 import de.tum.in.www1.artemis.domain.enumeration.ExerciseType;
 import de.tum.in.www1.artemis.domain.enumeration.IncludedInOverallScore;
 import de.tum.in.www1.artemis.domain.enumeration.InitializationState;
@@ -603,6 +604,10 @@ else if (resultDate1.isAfter(resultDate2)) {
     public Set<Result> findResultsFilteredForStudents(Participation participation) {
         boolean isAssessmentOver = getAssessmentDueDate() == null || getAssessmentDueDate().isBefore(ZonedDateTime.now());
         if (!isAssessmentOver) {
+            // This allows the showing of preliminary feedback in case the assessment due date is set before its over.
+            if (this instanceof TextExercise) {
+                return participation.getResults().stream().filter(result -> result.getAssessmentType() == AssessmentType.AUTOMATIC_ATHENA).collect(Collectors.toSet());
+            }
             return Set.of();
         }
         return participation.getResults().stream().filter(result -> result.getCompletionDate() != null).collect(Collectors.toSet());

src/main/java/de/tum/in/www1/artemis/domain/Submission.java

@@ -213,6 +213,16 @@ public List<Result> getManualResults() {
         return results.stream().filter(result -> result != null && !result.isAutomatic() && !result.isAthenaAutomatic()).collect(Collectors.toCollection(ArrayList::new));
     }
 
+    /**
+     * This method is necessary to ignore Athena results in the assessment view
+     *
+     * @return non athena automatic results including null results
+     */
+    @JsonIgnore
+    public List<Result> getNonAthenaResults() {
+        return results.stream().filter(result -> result == null || !result.isAthenaAutomatic()).collect(Collectors.toCollection(ArrayList::new));
+    }
+
     /**
      * Get the manual result by id of the submission
      *

src/main/java/de/tum/in/www1/artemis/service/ParticipationService.java

@@ -31,6 +31,7 @@
 import de.tum.in.www1.artemis.domain.enumeration.InitializationState;
 import de.tum.in.www1.artemis.domain.enumeration.SubmissionType;
 import de.tum.in.www1.artemis.domain.participation.Participant;
+import de.tum.in.www1.artemis.domain.participation.Participation;
 import de.tum.in.www1.artemis.domain.participation.ProgrammingExerciseStudentParticipation;
 import de.tum.in.www1.artemis.domain.participation.StudentParticipation;
 import de.tum.in.www1.artemis.domain.quiz.QuizExercise;
@@ -664,6 +665,21 @@ public Optional<StudentParticipation> findOneByExerciseAndStudentLoginWithEagerS
         return studentParticipationRepository.findWithEagerLegalSubmissionsByExerciseIdAndStudentLogin(exercise.getId(), username);
     }
 
+    /**
+     * Get one participation (in any state) by its student and exercise with eager submissions else throw exception.
+     *
+     * @param exercise the exercise for which to find a participation
+     * @param username the username of the student
+     * @return the participation of the given student and exercise with eager submissions in any state
+     */
+    public StudentParticipation findOneByExerciseAndStudentLoginWithEagerSubmissionsAnyStateElseThrow(Exercise exercise, String username) {
+        Optional<StudentParticipation> optionalParticipation = findOneByExerciseAndStudentLoginWithEagerSubmissionsAnyState(exercise, username);
+        if (optionalParticipation.isEmpty()) {
+            throw new EntityNotFoundException("No participation found in exercise with id " + exercise.getId() + " for user " + username);
+        }
+        return optionalParticipation.get();
+    }
+
     /**
      * Get all exercise participations belonging to exercise and student.
      *
@@ -693,6 +709,21 @@ public List<StudentParticipation> findByExerciseAndStudentIdWithEagerSubmissions
         return studentParticipationRepository.findByExerciseIdAndStudentIdWithEagerLegalSubmissions(exercise.getId(), studentId);
     }
 
+    /**
+     * Get the text exercise participation with the Latest Submissions and its results
+     *
+     * @param participationId the id of the participation
+     * @return the participation with latest submission and result
+     * @throws EntityNotFoundException
+     */
+    public StudentParticipation findTextExerciseParticipationWithLatestSubmissionAndResultElseThrow(Long participationId) throws EntityNotFoundException {
+        Optional<Participation> participation = participationRepository.findByIdWithLatestSubmissionAndResult(participationId);
+        if (participation.isEmpty() || !(participation.get() instanceof StudentParticipation studentParticipation)) {
+            throw new EntityNotFoundException("No text exercise participation found with id " + participationId);
+        }
+        return studentParticipation;
+    }
+
     /**
      * Get all programming exercise participations belonging to exercise and student with eager results and submissions.
      *

src/main/java/de/tum/in/www1/artemis/service/TextExerciseFeedbackService.java

@@ -0,0 +1,149 @@
+package de.tum.in.www1.artemis.service;
+
+import static de.tum.in.www1.artemis.config.Constants.PROFILE_CORE;
+
+import java.time.ZonedDateTime;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Service;
+
+import de.tum.in.www1.artemis.domain.Feedback;
+import de.tum.in.www1.artemis.domain.Result;
+import de.tum.in.www1.artemis.domain.TextExercise;
+import de.tum.in.www1.artemis.domain.TextSubmission;
+import de.tum.in.www1.artemis.domain.enumeration.AssessmentType;
+import de.tum.in.www1.artemis.domain.enumeration.FeedbackType;
+import de.tum.in.www1.artemis.domain.participation.Participation;
+import de.tum.in.www1.artemis.domain.participation.StudentParticipation;
+import de.tum.in.www1.artemis.repository.ResultRepository;
+import de.tum.in.www1.artemis.service.connectors.athena.AthenaFeedbackSuggestionsService;
+import de.tum.in.www1.artemis.web.rest.errors.BadRequestAlertException;
+import de.tum.in.www1.artemis.web.rest.errors.InternalServerErrorException;
+import de.tum.in.www1.artemis.web.websocket.ResultWebsocketService;
+
+@Profile(PROFILE_CORE)
+@Service
+public class TextExerciseFeedbackService {
+
+    private static final Logger log = LoggerFactory.getLogger(TextExerciseFeedbackService.class);
+
+    public static final String NON_GRADED_FEEDBACK_SUGGESTION = "NonGradedFeedbackSuggestion:";
+
+    private final Optional<AthenaFeedbackSuggestionsService> athenaFeedbackSuggestionsService;
+
+    private final ResultWebsocketService resultWebsocketService;
+
+    private final SubmissionService submissionService;
+
+    private final ParticipationService participationService;
+
+    private final ResultService resultService;
+
+    private final ResultRepository resultRepository;
+
+    public TextExerciseFeedbackService(Optional<AthenaFeedbackSuggestionsService> athenaFeedbackSuggestionsService, SubmissionService submissionService,
+            ResultService resultService, ResultRepository resultRepository, ResultWebsocketService resultWebsocketService, ParticipationService participationService) {
+        this.athenaFeedbackSuggestionsService = athenaFeedbackSuggestionsService;
+        this.submissionService = submissionService;
+        this.resultService = resultService;
+        this.resultRepository = resultRepository;
+        this.resultWebsocketService = resultWebsocketService;
+        this.participationService = participationService;
+    }
+
+    private void checkRateLimitOrThrow(StudentParticipation participation) {
+
+        List<Result> athenaResults = participation.getResults().stream().filter(result -> result.getAssessmentType() == AssessmentType.AUTOMATIC_ATHENA).toList();
+
+        long countOfAthenaResults = athenaResults.size();
+
+        if (countOfAthenaResults >= 10) {
+            throw new BadRequestAlertException("Maximum number of AI feedback requests reached.", "participation", "preconditions not met");
+        }
+    }
+
+    /**
+     * Handles the request for generating feedback for a text exercise.
+     * Unlike programming exercises a tutor is not notified if Athena is not available.
+     *
+     * @param exerciseId    the id of the text exercise.
+     * @param participation the student participation associated with the exercise.
+     * @param textExercise  the text exercise object.
+     * @return StudentParticipation updated text exercise for an AI assessment
+     */
+    public StudentParticipation handleNonGradedFeedbackRequest(Long exerciseId, StudentParticipation participation, TextExercise textExercise) {
+        if (this.athenaFeedbackSuggestionsService.isPresent()) {
+            this.checkRateLimitOrThrow(participation);
+            CompletableFuture.runAsync(() -> this.generateAutomaticNonGradedFeedback(participation, textExercise));
+        }
+        return participation;
+    }
+
+    /**
+     * Generates automatic non-graded feedback for a text exercise submission.
+     * This method leverages the Athena service to generate feedback based on the latest submission.
+     *
+     * @param participation the student participation associated with the exercise.
+     * @param textExercise  the text exercise object.
+     */
+    public void generateAutomaticNonGradedFeedback(StudentParticipation participation, TextExercise textExercise) {
+        log.debug("Using athena to generate (text exercise) feedback request: {}", textExercise.getId());
+
+        // athena takes over the control here
+        var submissionOptional = participationService.findTextExerciseParticipationWithLatestSubmissionAndResultElseThrow(participation.getId()).findLatestSubmission();
+
+        if (submissionOptional.isEmpty()) {
+            throw new BadRequestAlertException("No legal submissions found", "submission", "noSubmission");
+        }
+        var submission = submissionOptional.get();
+
+        Result automaticResult = new Result();
+        automaticResult.setAssessmentType(AssessmentType.AUTOMATIC_ATHENA);
+        automaticResult.setRated(true);
+        automaticResult.setScore(0.0);
+        automaticResult.setSuccessful(null);
+        automaticResult.setSubmission(submission);
+        automaticResult.setParticipation(participation);
+        try {
+            this.resultWebsocketService.broadcastNewResult((Participation) participation, automaticResult);
+
+            log.debug("Submission id: {}", submission.getId());
+
+            var athenaResponse = this.athenaFeedbackSuggestionsService.orElseThrow().getTextFeedbackSuggestions(textExercise, (TextSubmission) submission, false);
+
+            List<Feedback> feedbacks = athenaResponse.stream().filter(individualFeedbackItem -> individualFeedbackItem.description() != null).map(individualFeedbackItem -> {
+                var feedback = new Feedback();
+                feedback.setText(individualFeedbackItem.title());
+                feedback.setDetailText(individualFeedbackItem.description());
+                feedback.setHasLongFeedbackText(false);
+                feedback.setType(FeedbackType.AUTOMATIC);
+                feedback.setCredits(individualFeedbackItem.credits());
+                return feedback;
+            }).toList();
+
+            double totalFeedbacksScore = 0.0;
+            for (Feedback feedback : feedbacks) {
+                totalFeedbacksScore += feedback.getCredits();
+            }
+            totalFeedbacksScore = totalFeedbacksScore / textExercise.getMaxPoints() * 100;
+            automaticResult.setSuccessful(true);
+            automaticResult.setCompletionDate(ZonedDateTime.now());
+
+            automaticResult.setScore(Math.clamp(totalFeedbacksScore, 0, 100));
+
+            automaticResult = this.resultRepository.save(automaticResult);
+            resultService.storeFeedbackInResult(automaticResult, feedbacks, true);
+            submissionService.saveNewResult(submission, automaticResult);
+            this.resultWebsocketService.broadcastNewResult((Participation) participation, automaticResult);
+        }
+        catch (Exception e) {
+            log.error("Could not generate feedback", e);
+            throw new InternalServerErrorException("Something went wrong... AI Feedback could not be generated");
+        }
+    }
+}

src/main/java/de/tum/in/www1/artemis/service/TextSubmissionService.java

@@ -81,6 +81,13 @@ public TextSubmission handleTextSubmission(TextSubmission textSubmission, TextEx
         if (exercise.isExamExercise() || exerciseDateService.isBeforeDueDate(participation)) {
             textSubmission.setSubmitted(true);
         }
+
+        // if athena results are present than create new submission on submit
+        if (!textSubmission.getResults().isEmpty()) {
+            log.debug("Creating a new submission due to Athena results for user: {}", user.getLogin());
+            textSubmission.setId(null);
+        }
+
         textSubmission = save(textSubmission, participation, exercise, user);
         return textSubmission;
     }
@@ -104,7 +111,6 @@ private TextSubmission save(TextSubmission textSubmission, StudentParticipation
             participation.setInitializationState(InitializationState.FINISHED);
             studentParticipationRepository.save(participation);
         }
-
         // remove result from submission (in the unlikely case it is passed here), so that students cannot inject a result
         textSubmission.setResults(new ArrayList<>());
         textSubmission = textSubmissionRepository.save(textSubmission);

src/main/java/de/tum/in/www1/artemis/web/rest/AbstractSubmissionResource.java

@@ -81,6 +81,10 @@ protected ResponseEntity<List<Submission>> getAllSubmissions(Long exerciseId, bo
             if (submission.getParticipation() != null && submission.getParticipation().getExercise() != null) {
                 submission.getParticipation().setExercise(null);
             }
+            // Important for exercises with Athena results
+            if (assessedByTutor) {
+                submission.setResults(submission.getNonAthenaResults());
+            }
         });
 
         return ResponseEntity.ok().body(submissions);