Avoid generating duplicate competencies

app/domain/competency_extraction_pipeline_execution_dto.py

@@ -1,14 +1,17 @@
-from typing import List, Optional
+from typing import List
 
 from pydantic import Field, BaseModel
 
 from . import PipelineExecutionDTO
-from .data.competency_dto import CompetencyTaxonomy
+from .data.competency_dto import CompetencyTaxonomy, Competency
 
 
 class CompetencyExtractionPipelineExecutionDTO(BaseModel):
     execution: PipelineExecutionDTO
-    course_description: Optional[str] = Field(alias="courseDescription")
+    course_description: str = Field(alias="courseDescription")
+    current_competencies: list[Competency] = Field(
+        alias="currentCompetencies", default=[]
+    )
     taxonomy_options: List[CompetencyTaxonomy] = Field(
         alias="taxonomyOptions", default=[]
     )

app/pipeline/competency_extraction_pipeline.py

@@ -48,11 +48,20 @@ def __call__(
             raise ValueError("Non-zero max_n is required")
 
         taxonomy_options = ", ".join(dto.taxonomy_options)
+        current_competencies = "\n\n".join(
+            [c.model_dump_json(indent=4) for c in dto.current_competencies]
+        )
+        if current_competencies:
+            current_competencies = (
+                f"\nHere are the current competencies in the course:\n{current_competencies}\n"
+                f"Do not repeat these competencies.\n"
+            )
 
         prompt = system_prompt.format(
             taxonomy_list=taxonomy_options,
             course_description=dto.course_description,
             max_n=dto.max_n,
+            current_competencies=current_competencies,
         )
         prompt = PyrisMessage(
             sender=IrisMessageRole.SYSTEM,

app/pipeline/prompts/competency_extraction.py

@@ -10,9 +10,9 @@
 The title of the competency, which is a specific topic or skill. This should be a short phrase of at most 4 words.
 
 - description:
-A detailed description of the competency in 3 to 5 bullet points.
+A detailed description of the competency in 2 to 5 bullet points.
 Each bullet point illustrates a specific skill or concept of the competency.
-Each bullet point is a complete sentence starting with "You" and containing at most 15 words.
+Each bullet point is a complete sentence containing at most 15 words.
 Each bullet point is on a new line and starts with "- ".
 
 - taxonomy:
@@ -27,6 +27,8 @@
 - is small and fine-grained. Large topics should be broken down into smaller competencies.
 - does not overlap with other competencies: each competency is unique. Expanding on a previous competency is allowed.
 
+Here is the provided course description: {course_description}
+
 Here is a template competency in JSON format:
 
 {{
@@ -35,8 +37,8 @@
     "taxonomy": "ANALYZE"
 }}
 
-Here is the provided course description: {course_description}
+{current_competencies}
 
-Respond with up to {max_n} competencies extracted from the course description,
+Respond with 0 to {max_n} competencies extracted from the course description,
 each in JSON format, split by two newlines.
 """