Skip to content

Commit

Permalink
Programming exercises: Add visualization of test case errors (#9213)
Browse files Browse the repository at this point in the history
  • Loading branch information
az108 authored Sep 7, 2024
1 parent b7aaf44 commit fa2e539
Show file tree
Hide file tree
Showing 15 changed files with 458 additions and 16 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import de.tum.in.www1.artemis.domain.participation.StudentParticipation;
import de.tum.in.www1.artemis.domain.quiz.QuizSubmittedAnswerCount;
import de.tum.in.www1.artemis.repository.base.ArtemisJpaRepository;
import de.tum.in.www1.artemis.web.rest.dto.feedback.FeedbackDetailDTO;

/**
* Spring Data JPA repository for the Participation entity.
Expand Down Expand Up @@ -1230,4 +1231,53 @@ SELECT COALESCE(AVG(p.presentationScore), 0)
AND p.presentationScore IS NOT NULL
""")
double getAvgPresentationScoreByCourseId(@Param("courseId") long courseId);

/**
* Retrieves aggregated feedback details for a given exercise, including the count of each unique feedback detail text and test case name.
* <br>
* The relative count and task number are initially set to 0 and are calculated in a separate step in the service layer.
*
* @param exerciseId Exercise ID.
* @return a list of {@link FeedbackDetailDTO} objects, with the relative count and task number set to 0.
*/
@Query("""
SELECT new de.tum.in.www1.artemis.web.rest.dto.feedback.FeedbackDetailDTO(
COUNT(f.id),
0,
f.detailText,
f.testCase.testName,
0
)
FROM StudentParticipation p
JOIN p.results r
JOIN r.feedbacks f
WHERE p.exercise.id = :exerciseId
AND p.testRun = FALSE
AND r.id = (
SELECT MAX(pr.id)
FROM p.results pr
)
AND f.positive = FALSE
GROUP BY f.detailText, f.testCase.testName
""")
List<FeedbackDetailDTO> findAggregatedFeedbackByExerciseId(@Param("exerciseId") long exerciseId);

/**
* Counts the distinct number of latest results for a given exercise, excluding those in practice mode.
*
* @param exerciseId Exercise ID.
* @return The count of distinct latest results for the exercise.
*/
@Query("""
SELECT COUNT(DISTINCT r.id)
FROM StudentParticipation p
JOIN p.results r
WHERE p.exercise.id = :exerciseId
AND p.testRun = FALSE
AND r.id = (
SELECT MAX(pr.id)
FROM p.results pr
)
""")
long countDistinctResultsByExerciseId(@Param("exerciseId") long exerciseId);
}
43 changes: 42 additions & 1 deletion src/main/java/de/tum/in/www1/artemis/service/ResultService.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import de.tum.in.www1.artemis.domain.enumeration.BuildPlanType;
import de.tum.in.www1.artemis.domain.enumeration.FeedbackType;
import de.tum.in.www1.artemis.domain.exam.Exam;
import de.tum.in.www1.artemis.domain.hestia.ProgrammingExerciseTask;
import de.tum.in.www1.artemis.domain.participation.Participation;
import de.tum.in.www1.artemis.domain.participation.ProgrammingExerciseParticipation;
import de.tum.in.www1.artemis.domain.participation.ProgrammingExerciseStudentParticipation;
Expand All @@ -49,11 +50,15 @@
import de.tum.in.www1.artemis.repository.ResultRepository;
import de.tum.in.www1.artemis.repository.SolutionProgrammingExerciseParticipationRepository;
import de.tum.in.www1.artemis.repository.StudentExamRepository;
import de.tum.in.www1.artemis.repository.StudentParticipationRepository;
import de.tum.in.www1.artemis.repository.TemplateProgrammingExerciseParticipationRepository;
import de.tum.in.www1.artemis.repository.UserRepository;
import de.tum.in.www1.artemis.repository.hestia.ProgrammingExerciseTaskRepository;
import de.tum.in.www1.artemis.security.Role;
import de.tum.in.www1.artemis.service.connectors.localci.dto.ResultBuildJob;
import de.tum.in.www1.artemis.service.connectors.lti.LtiNewResultService;
import de.tum.in.www1.artemis.service.hestia.ProgrammingExerciseTaskService;
import de.tum.in.www1.artemis.web.rest.dto.feedback.FeedbackDetailDTO;
import de.tum.in.www1.artemis.web.rest.errors.BadRequestAlertException;
import de.tum.in.www1.artemis.web.websocket.ResultWebsocketService;

Expand Down Expand Up @@ -99,14 +104,19 @@ public class ResultService {

private final BuildLogEntryService buildLogEntryService;

private final StudentParticipationRepository studentParticipationRepository;

private final ProgrammingExerciseTaskService programmingExerciseTaskService;

public ResultService(UserRepository userRepository, ResultRepository resultRepository, Optional<LtiNewResultService> ltiNewResultService,
ResultWebsocketService resultWebsocketService, ComplaintResponseRepository complaintResponseRepository, RatingRepository ratingRepository,
FeedbackRepository feedbackRepository, LongFeedbackTextRepository longFeedbackTextRepository, ComplaintRepository complaintRepository,
ParticipantScoreRepository participantScoreRepository, AuthorizationCheckService authCheckService, ExerciseDateService exerciseDateService,
TemplateProgrammingExerciseParticipationRepository templateProgrammingExerciseParticipationRepository,
SolutionProgrammingExerciseParticipationRepository solutionProgrammingExerciseParticipationRepository,
ProgrammingExerciseStudentParticipationRepository programmingExerciseStudentParticipationRepository, StudentExamRepository studentExamRepository,
BuildJobRepository buildJobRepository, BuildLogEntryService buildLogEntryService) {
BuildJobRepository buildJobRepository, BuildLogEntryService buildLogEntryService, StudentParticipationRepository studentParticipationRepository,
ProgrammingExerciseTaskRepository programmingExerciseTaskRepository, ProgrammingExerciseTaskService programmingExerciseTaskService) {
this.userRepository = userRepository;
this.resultRepository = resultRepository;
this.ltiNewResultService = ltiNewResultService;
Expand All @@ -125,6 +135,8 @@ public ResultService(UserRepository userRepository, ResultRepository resultRepos
this.studentExamRepository = studentExamRepository;
this.buildJobRepository = buildJobRepository;
this.buildLogEntryService = buildLogEntryService;
this.studentParticipationRepository = studentParticipationRepository;
this.programmingExerciseTaskService = programmingExerciseTaskService;
}

/**
Expand Down Expand Up @@ -513,4 +525,33 @@ private Result shouldSaveResult(@NotNull Result result, boolean shouldSave) {
return result;
}
}

/**
* Retrieves aggregated feedback details for a given exercise, calculating relative counts based on the total number of distinct results.
* The task numbers are assigned based on the associated test case names, using the set of tasks fetched from the database.
* <br>
* For each feedback detail:
* 1. The relative count is calculated as a percentage of the total number of distinct results for the exercise.
* 2. The task number is determined by matching the test case name with the tasks.
*
* @param exerciseId The ID of the exercise for which feedback details should be retrieved.
* @return A list of FeedbackDetailDTO objects, each containing:
* - feedback count,
* - relative count (as a percentage of distinct results),
* - detail text,
* - test case name,
* - determined task number (based on the test case name).
*/
public List<FeedbackDetailDTO> findAggregatedFeedbackByExerciseId(long exerciseId) {
long distinctResultCount = studentParticipationRepository.countDistinctResultsByExerciseId(exerciseId);
Set<ProgrammingExerciseTask> tasks = programmingExerciseTaskService.getTasksWithUnassignedTestCases(exerciseId);
List<FeedbackDetailDTO> feedbackDetails = studentParticipationRepository.findAggregatedFeedbackByExerciseId(exerciseId);

return feedbackDetails.stream().map(detail -> {
double relativeCount = (detail.count() * 100.0) / distinctResultCount;
int taskNumber = tasks.stream().filter(task -> task.getTestCases().stream().anyMatch(tc -> tc.getTestName().equals(detail.testCaseName()))).findFirst()
.map(task -> tasks.stream().toList().indexOf(task) + 1).orElse(0);
return new FeedbackDetailDTO(detail.count(), relativeCount, detail.detailText(), detail.testCaseName(), taskNumber);
}).toList();
}
}
16 changes: 16 additions & 0 deletions src/main/java/de/tum/in/www1/artemis/web/rest/ResultResource.java
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,14 @@
import de.tum.in.www1.artemis.security.annotations.EnforceAtLeastInstructor;
import de.tum.in.www1.artemis.security.annotations.EnforceAtLeastStudent;
import de.tum.in.www1.artemis.security.annotations.EnforceAtLeastTutor;
import de.tum.in.www1.artemis.security.annotations.enforceRoleInExercise.EnforceAtLeastEditorInExercise;
import de.tum.in.www1.artemis.service.AuthorizationCheckService;
import de.tum.in.www1.artemis.service.ParticipationAuthorizationCheckService;
import de.tum.in.www1.artemis.service.ParticipationService;
import de.tum.in.www1.artemis.service.ResultService;
import de.tum.in.www1.artemis.service.exam.ExamDateService;
import de.tum.in.www1.artemis.web.rest.dto.ResultWithPointsPerGradingCriterionDTO;
import de.tum.in.www1.artemis.web.rest.dto.feedback.FeedbackDetailDTO;
import de.tum.in.www1.artemis.web.rest.errors.BadRequestAlertException;
import de.tum.in.www1.artemis.web.rest.util.HeaderUtil;

Expand Down Expand Up @@ -276,4 +278,18 @@ public ResponseEntity<Result> createResultForExternalSubmission(@PathVariable Lo
return ResponseEntity.created(new URI("/api/results/" + savedResult.getId()))
.headers(HeaderUtil.createEntityCreationAlert(applicationName, true, ENTITY_NAME, savedResult.getId().toString())).body(savedResult);
}

/**
* GET /exercises/:exerciseId/feedback-details : Retrieves all aggregated feedback details for a given exercise.
* The feedback details include counts and relative counts of feedback occurrences, along with associated test case names and task numbers.
*
* @param exerciseId The ID of the exercise for which feedback details should be retrieved.
* @return A ResponseEntity containing a list of {@link FeedbackDetailDTO}s
*/
@GetMapping("exercises/{exerciseId}/feedback-details")
@EnforceAtLeastEditorInExercise
public ResponseEntity<List<FeedbackDetailDTO>> getAllFeedbackDetailsForExercise(@PathVariable Long exerciseId) {
log.debug("REST request to get all Feedback details for Exercise {}", exerciseId);
return ResponseEntity.ok(resultService.findAggregatedFeedbackByExerciseId(exerciseId));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package de.tum.in.www1.artemis.web.rest.dto.feedback;

import com.fasterxml.jackson.annotation.JsonInclude;

@JsonInclude(JsonInclude.Include.NON_EMPTY)
public record FeedbackDetailDTO(long count, double relativeCount, String detailText, String testCaseName, int taskNumber) {
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
<div class="m-3">
<h2 class="mb-3" jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.title" [translateValues]="{ exerciseTitle: exerciseTitle }"></h2>
<table class="table table-striped mb-3">
<thead>
<tr>
<th scope="col" jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.occurrence"></th>
<th scope="col" jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.feedback"></th>
<th scope="col" jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.task"></th>
<th scope="col" jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.testcase"></th>
<th scope="col" jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.errorCategory"></th>
</tr>
</thead>
<tbody class="table-group-divider">
@for (item of feedbackDetails; track item) {
<tr>
<td class="text-center">{{ item.count }} ({{ item.relativeCount | number: '1.0-0' }}%)</td>
<td>{{ item.detailText }}</td>
<td class="text-center">{{ item.taskNumber }}</td>
<td>{{ item.testCaseName }}</td>
<td>Student Error</td>
<!-- This is a placeholder, will be covered in follow up PRs -->
</tr>
}
</tbody>
</table>
<div jhiTranslate="artemisApp.programmingExercise.configureGrading.feedbackAnalysis.totalItems" [translateValues]="{ count: feedbackDetails.length }"></div>
</div>
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import { Component, Input, OnInit } from '@angular/core';
import { FeedbackAnalysisService, FeedbackDetail } from 'app/exercises/programming/manage/grading/feedback-analysis/feedback-analysis.service';
import { ArtemisSharedModule } from 'app/shared/shared.module';
import { AlertService } from 'app/core/util/alert.service';

@Component({
selector: 'jhi-feedback-analysis',
templateUrl: './feedback-analysis.component.html',
standalone: true,
imports: [ArtemisSharedModule],
providers: [FeedbackAnalysisService],
})
export class FeedbackAnalysisComponent implements OnInit {
@Input() exerciseTitle: string;
@Input() exerciseId: number;
feedbackDetails: FeedbackDetail[] = [];

constructor(
private feedbackAnalysisService: FeedbackAnalysisService,
private alertService: AlertService,
) {}

ngOnInit(): void {
this.loadFeedbackDetails(this.exerciseId);
}

async loadFeedbackDetails(exerciseId: number): Promise<void> {
try {
this.feedbackDetails = await this.feedbackAnalysisService.getFeedbackDetailsForExercise(exerciseId);
} catch (error) {
this.alertService.error(`artemisApp.programmingExercise.configureGrading.feedbackAnalysis.error`);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { Injectable } from '@angular/core';
import { BaseApiHttpService } from 'app/course/learning-paths/services/base-api-http.service';

export interface FeedbackDetail {
count: number;
relativeCount: number;
detailText: string;
testCaseName: string;
taskNumber: number;
}

@Injectable()
export class FeedbackAnalysisService extends BaseApiHttpService {
private readonly EXERCISE_RESOURCE_URL = 'exercises';

getFeedbackDetailsForExercise(exerciseId: number): Promise<FeedbackDetail[]> {
return this.get<FeedbackDetail[]>(`${this.EXERCISE_RESOURCE_URL}/${exerciseId}/feedback-details`);
}
}
Original file line number Diff line number Diff line change
@@ -1,35 +1,48 @@
<ng-template #tabTemplate let-type="type" let-translation="translation">
<div class="tab-item" (click)="selectTab(type)" [ngClass]="activeTab === type ? 'active' : ''">
<b [jhiTranslate]="translation"></b>
</div>
</ng-template>
<div>
<div class="d-flex align-content-center mb-2">
<h3 class="fw-medium" jhiTranslate="artemisApp.programmingExercise.configureGrading.title"></h3>
</div>
@if (!isLoading) {
<div class="top-bar">
<div class="d-flex align-items-center">
<div class="tab-item test-cases" (click)="selectTab('test-cases')" [ngClass]="activeTab === 'test-cases' ? 'active' : ''">
<b>Test Cases</b>
</div>
<ng-container
*ngTemplateOutlet="tabTemplate; context: { type: 'test-cases', translation: 'artemisApp.programmingExercise.configureGrading.testCases.title' }"
></ng-container>
@if (programmingExercise.staticCodeAnalysisEnabled) {
<div class="tab-item code-analysis" (click)="selectTab('code-analysis')" [ngClass]="activeTab === 'code-analysis' ? 'active' : ''">
<b>Code Analysis</b>
</div>
<ng-container
*ngTemplateOutlet="tabTemplate; context: { type: 'code-analysis', translation: 'artemisApp.programmingExercise.configureGrading.categories.titleHeader' }"
></ng-container>
}
<ng-container
*ngTemplateOutlet="tabTemplate; context: { type: 'submission-policy', translation: 'artemisApp.programmingExercise.submissionPolicy.title' }"
></ng-container>
@if (programmingExercise.isAtLeastEditor) {
<ng-container
*ngTemplateOutlet="
tabTemplate;
context: { type: 'feedback-analysis', translation: 'artemisApp.programmingExercise.configureGrading.feedbackAnalysis.titleHeader' }
"
></ng-container>
}
<div class="tab-item submission-policy" (click)="selectTab('submission-policy')" [ngClass]="activeTab === 'submission-policy' ? 'active' : ''">
<b>Submission Policy</b>
</div>
</div>
<ng-template>
<div></div>
</ng-template>
<div class="d-flex align-items-center">
@if (activeTab !== 'submission-policy') {
@if (activeTab === 'test-cases' || activeTab === 'code-analysis') {
<jhi-programming-exercise-configure-grading-status
[exerciseIsReleasedAndHasResults]="isReleasedAndHasResults"
[hasUnsavedTestCaseChanges]="hasUnsavedChanges"
[hasUnsavedCategoryChanges]="!!changedCategoryIds.length"
[hasUpdatedGradingConfig]="hasUpdatedGradingConfig"
/>
}
@if (programmingExercise.isAtLeastInstructor) {
@if (programmingExercise.isAtLeastInstructor && activeTab !== 'feedback-analysis') {
<jhi-programming-exercise-configure-grading-actions
[exercise]="programmingExercise"
[hasUpdatedGradingConfig]="hasUpdatedGradingConfig"
Expand Down Expand Up @@ -256,5 +269,10 @@ <h2 class="mb-5 fw-medium">
</div>
}
</div>
<div class="grading-body-container mt-3">
@if (programmingExercise.isAtLeastEditor && activeTab === 'feedback-analysis') {
<jhi-feedback-analysis [exerciseTitle]="programmingExercise.title!" [exerciseId]="programmingExercise.id!"></jhi-feedback-analysis>
}
</div>
}
</div>
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,7 @@ const DefaultFieldValues: { [key: string]: number } = {
[EditableField.MAX_PENALTY]: 0,
};

export type GradingTab = 'test-cases' | 'code-analysis' | 'submission-policy';

export type GradingTab = 'test-cases' | 'code-analysis' | 'submission-policy' | 'feedback-analysis';
export type Table = 'testCases' | 'codeAnalysis';

@Component({
Expand Down Expand Up @@ -232,7 +231,8 @@ export class ProgrammingExerciseConfigureGradingComponent implements OnInit, OnD
this.isLoading = false;
}

if (params['tab'] === 'test-cases' || params['tab'] === 'code-analysis' || params['tab'] === 'submission-policy') {
const gradingTabs: GradingTab[] = ['test-cases', 'code-analysis', 'submission-policy', 'feedback-analysis'];
if (gradingTabs.includes(params['tab'])) {
this.selectTab(params['tab']);
} else {
this.selectTab('test-cases');
Expand Down
Loading

0 comments on commit fa2e539

Please sign in to comment.