From cc8956d96a5dfb794e4e40a47db36e8e9a731b79 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Tue, 9 Jul 2024 14:36:26 +0200 Subject: [PATCH 1/9] Fix Bugs fixes found on Sentry --- app/pipeline/chat/exercise_chat_pipeline.py | 2 +- app/pipeline/prompts/reranker_prompt.txt | 2 +- app/pipeline/shared/reranker_pipeline.py | 2 +- app/retrieval/lecture_retrieval.py | 5 +++-- app/web/status/status_update.py | 4 ++-- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/pipeline/chat/exercise_chat_pipeline.py b/app/pipeline/chat/exercise_chat_pipeline.py index 3f87f68..efea3d1 100644 --- a/app/pipeline/chat/exercise_chat_pipeline.py +++ b/app/pipeline/chat/exercise_chat_pipeline.py @@ -346,7 +346,7 @@ def _add_relevant_chunks_to_prompt(self, retrieved_lecture_chunks: List[dict]): ) txt += lct - self.prompt += SystemMessagePromptTemplate.from_template(txt) + self.prompt += SystemMessagePromptTemplate.from_template(txt.replace("{", "{{").replace("}", "}}")) def should_execute_lecture_pipeline(self, course_id: int) -> bool: """ diff --git a/app/pipeline/prompts/reranker_prompt.txt b/app/pipeline/prompts/reranker_prompt.txt index c682c42..7bfea83 100644 --- a/app/pipeline/prompts/reranker_prompt.txt +++ b/app/pipeline/prompts/reranker_prompt.txt @@ -3,7 +3,7 @@ Respond with the numbers of the paragraphs you should consult to answer the ques The relevance score is a number from 1 to 10 based on how relevant the paragraphs are to answer the question. Do not include any paragraphs that are not relevant to the question. Without any comment, return the result in the following JSON format, it is important to avoid giving -unnecessary information, only the number of the paragraph if it's really necessary for answering the student's question +unnecessary information, only the number of the paragraph if it's necessary for answering the student's question otherwise leave the array empty. {{"selected_paragraphs": [, , ...]}} diff --git a/app/pipeline/shared/reranker_pipeline.py b/app/pipeline/shared/reranker_pipeline.py index 178bb4e..0ca2581 100644 --- a/app/pipeline/shared/reranker_pipeline.py +++ b/app/pipeline/shared/reranker_pipeline.py @@ -29,7 +29,7 @@ def __init__(self): super().__init__(implementation_id="reranker_pipeline") request_handler = CapabilityRequestHandler( requirements=RequirementList( - gpt_version_equivalent=3.5, + gpt_version_equivalent=4, context_length=16385, ) ) diff --git a/app/retrieval/lecture_retrieval.py b/app/retrieval/lecture_retrieval.py index 566acb8..df832eb 100644 --- a/app/retrieval/lecture_retrieval.py +++ b/app/retrieval/lecture_retrieval.py @@ -143,7 +143,8 @@ def __call__( selected_chunks_index = self.reranker_pipeline( paragraphs=merged_chunks, query=student_query, chat_history=chat_history ) - return [merged_chunks[int(i)] for i in selected_chunks_index] + if selected_chunks_index: + return [merged_chunks[int(i)] for i in selected_chunks_index] return [] @traceable(name="Basic Lecture Retrieval") @@ -467,7 +468,7 @@ def run_parallel_rewrite_tasks( response_future = executor.submit( self.search_in_db, query=rewritten_query, - hybrid_factor=0.7, + hybrid_factor=0.9, result_limit=result_limit, course_id=course_id, base_url=base_url, diff --git a/app/web/status/status_update.py b/app/web/status/status_update.py index 6ab91f4..533047c 100644 --- a/app/web/status/status_update.py +++ b/app/web/status/status_update.py @@ -127,7 +127,7 @@ def error(self, message: str, exception=None): """ self.stage.state = StageStateEnum.ERROR self.stage.message = message - self.stage.result = None + self.status.result = None self.stage.suggestions = None # Set all subsequent stages to SKIPPED if an error occurs rest_of_index = ( @@ -157,7 +157,7 @@ def skip(self, message: Optional[str] = None, start_next_stage: bool = True): """ self.stage.state = StageStateEnum.SKIPPED self.stage.message = message - self.stage.result = None + self.status.result = None self.stage.suggestions = None next_stage = self.get_next_stage() if next_stage is not None: From d598376cb203f6b77da24b1842ad1599f5ba5b0b Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Tue, 9 Jul 2024 14:36:53 +0200 Subject: [PATCH 2/9] Lint --- app/pipeline/chat/exercise_chat_pipeline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/pipeline/chat/exercise_chat_pipeline.py b/app/pipeline/chat/exercise_chat_pipeline.py index efea3d1..8043e9a 100644 --- a/app/pipeline/chat/exercise_chat_pipeline.py +++ b/app/pipeline/chat/exercise_chat_pipeline.py @@ -346,7 +346,9 @@ def _add_relevant_chunks_to_prompt(self, retrieved_lecture_chunks: List[dict]): ) txt += lct - self.prompt += SystemMessagePromptTemplate.from_template(txt.replace("{", "{{").replace("}", "}}")) + self.prompt += SystemMessagePromptTemplate.from_template( + txt.replace("{", "{{").replace("}", "}}") + ) def should_execute_lecture_pipeline(self, course_id: int) -> bool: """ From b93781cf27764cd1bf3de8d66d7b0c670d28ca46 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Wed, 10 Jul 2024 14:08:24 +0200 Subject: [PATCH 3/9] revert gpt to 3.5 --- app/pipeline/shared/reranker_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/pipeline/shared/reranker_pipeline.py b/app/pipeline/shared/reranker_pipeline.py index 0ca2581..178bb4e 100644 --- a/app/pipeline/shared/reranker_pipeline.py +++ b/app/pipeline/shared/reranker_pipeline.py @@ -29,7 +29,7 @@ def __init__(self): super().__init__(implementation_id="reranker_pipeline") request_handler = CapabilityRequestHandler( requirements=RequirementList( - gpt_version_equivalent=4, + gpt_version_equivalent=3.5, context_length=16385, ) ) From f91d377c9c8ddd55533e390666cf40bfdf6fd266 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Thu, 11 Jul 2024 12:57:24 +0200 Subject: [PATCH 4/9] Add links to citations --- app/domain/data/lecture_unit_dto.py | 1 + app/pipeline/chat/lecture_chat_pipeline.py | 5 +++-- app/pipeline/lecture_ingestion_pipeline.py | 1 + app/pipeline/prompts/citation_prompt.txt | 11 ++++++----- app/pipeline/shared/citation_pipeline.py | 5 +++-- app/retrieval/lecture_retrieval.py | 8 ++++---- app/vector_database/lecture_schema.py | 7 +++++++ 7 files changed, 25 insertions(+), 13 deletions(-) diff --git a/app/domain/data/lecture_unit_dto.py b/app/domain/data/lecture_unit_dto.py index 26ef178..1396b82 100644 --- a/app/domain/data/lecture_unit_dto.py +++ b/app/domain/data/lecture_unit_dto.py @@ -7,6 +7,7 @@ class LectureUnitDTO(BaseModel): pdf_file_base64: str = Field(default="", alias="pdfFile") lecture_unit_id: int = Field(alias="lectureUnitId") lecture_unit_name: str = Field(default="", alias="lectureUnitName") + lecture_unit_link: str = Field(default="", alias="lectureUnitLink") lecture_id: int = Field(alias="lectureId") lecture_name: str = Field(default="", alias="lectureName") course_id: int = Field(alias="courseId") diff --git a/app/pipeline/chat/lecture_chat_pipeline.py b/app/pipeline/chat/lecture_chat_pipeline.py index 5169300..297a758 100644 --- a/app/pipeline/chat/lecture_chat_pipeline.py +++ b/app/pipeline/chat/lecture_chat_pipeline.py @@ -46,7 +46,8 @@ def lecture_initial_prompt(): questions about the lectures. To answer them the best way, relevant lecture content is provided to you with the student's question. If the context provided to you is not enough to formulate an answer to the student question you can simply ask the student to elaborate more on his question. Use only the parts of the context provided for - you that is relevant to the student's question. If the user greets you greet him back, and ask him how you can help + you that is relevant to the student's question. If the user greets you greet him back, and ask him how you can help. + Always formulate your answer in the same language as the user's language. """ @@ -103,7 +104,7 @@ def __call__(self, dto: LectureChatPipelineExecutionDTO): retrieved_lecture_chunks = self.retriever( chat_history=history, student_query=query.contents[0].text_content, - result_limit=10, + result_limit=5, course_name=dto.course.name, course_id=dto.course.id, base_url=dto.settings.artemis_base_url, diff --git a/app/pipeline/lecture_ingestion_pipeline.py b/app/pipeline/lecture_ingestion_pipeline.py index 0b468a4..d7a5dd0 100644 --- a/app/pipeline/lecture_ingestion_pipeline.py +++ b/app/pipeline/lecture_ingestion_pipeline.py @@ -73,6 +73,7 @@ def create_page_data( LectureSchema.LECTURE_NAME.value: lecture_unit_dto.lecture_name, LectureSchema.LECTURE_UNIT_ID.value: lecture_unit_dto.lecture_unit_id, LectureSchema.LECTURE_UNIT_NAME.value: lecture_unit_dto.lecture_unit_name, + LectureSchema.LECTURE_UNIT_LINK.value: lecture_unit_dto.lecture_unit_link, LectureSchema.COURSE_ID.value: lecture_unit_dto.course_id, LectureSchema.COURSE_NAME.value: lecture_unit_dto.course_name, LectureSchema.COURSE_DESCRIPTION.value: lecture_unit_dto.course_description, diff --git a/app/pipeline/prompts/citation_prompt.txt b/app/pipeline/prompts/citation_prompt.txt index e62c2ac..375278f 100644 --- a/app/pipeline/prompts/citation_prompt.txt +++ b/app/pipeline/prompts/citation_prompt.txt @@ -1,8 +1,9 @@ In the paragraphs below you are provided with an answer to a question. Underneath the answer you will find the paragraphs that the answer was based on. Add citations of the paragraphs to the answer. Cite the paragraphs in brackets after the sentence where the information is used in the answer. -At the end of the answer list each source with its corresponding number and provide the Lecture Title,as well as the page number in this format "[1] Lecture title, page number". +At the end of the answer, list each source with its corresponding number and provide the Lecture Title, page number, and a clickable link in this format: [1] "Lecture title", "page number". +If the answer uses multiple pages from the same lecture, list the page numbers at the same line separated by commas in this format : [1] "Lecture title", "page number1,number2,number3". Do not Include the Actual paragraphs, only the citations at the end. -Only include the paragraphs that are relevant to the answer. +Only include the citations of the paragraphs that are relevant to the answer. If the answer actually does not contain any information from the paragraphs, please do not include any citations and return '!NONE!'. But if the answer contains information from the paragraphs, ALWAYS include citations. @@ -10,8 +11,8 @@ Here is an example how to rewrite the answer with citations (note the empty line " Lorem ipsum dolor sit amet, consectetur adipiscing elit [1]. Ded do eiusmod tempor incididunt ut labore et dolore magna aliqua [2]. -[1] Lecture 1, page 2. -[2] Lecture 2, page 5. +[1] Lecture 1, page 2,3,4. +[2] Lecture 2, page 5,25. " Here are the answer and the paragraphs: @@ -19,7 +20,7 @@ Here are the answer and the paragraphs: Answer without citations: {Answer} -Paragraphs: +Paragraphs with their Lecture Names, Links and Page Numbers: {Paragraphs} Answer with citations (ensure empty line between the message and the citations): diff --git a/app/pipeline/shared/citation_pipeline.py b/app/pipeline/shared/citation_pipeline.py index be73b98..4761b9c 100644 --- a/app/pipeline/shared/citation_pipeline.py +++ b/app/pipeline/shared/citation_pipeline.py @@ -25,7 +25,7 @@ def __init__(self): super().__init__(implementation_id="citation_pipeline") request_handler = CapabilityRequestHandler( requirements=RequirementList( - gpt_version_equivalent=4.5, + gpt_version_equivalent=3.5, context_length=16385, ) ) @@ -52,7 +52,8 @@ def create_formatted_string(self, paragraphs): formatted_string = "" for i, paragraph in enumerate(paragraphs): lct = "Lecture: {}, Page: {}\nContent:\n---{}---\n\n".format( - paragraph.get(LectureSchema.LECTURE_NAME.value), + paragraph.get(LectureSchema.LECTURE_UNIT_NAME.value), + paragraph.get(LectureSchema.LECTURE_UNIT_LINK.value), paragraph.get(LectureSchema.PAGE_NUMBER.value), paragraph.get(LectureSchema.PAGE_TEXT_CONTENT.value), ) diff --git a/app/retrieval/lecture_retrieval.py b/app/retrieval/lecture_retrieval.py index df832eb..89ff8dd 100644 --- a/app/retrieval/lecture_retrieval.py +++ b/app/retrieval/lecture_retrieval.py @@ -387,11 +387,11 @@ def search_in_db( alpha=hybrid_factor, vector=vec, return_properties=[ - LectureSchema.PAGE_TEXT_CONTENT.value, - LectureSchema.COURSE_NAME.value, - LectureSchema.LECTURE_NAME.value, - LectureSchema.PAGE_NUMBER.value, LectureSchema.COURSE_ID.value, + LectureSchema.LECTURE_UNIT_NAME.value, + LectureSchema.LECTURE_UNIT_LINK.value, + LectureSchema.PAGE_NUMBER.value, + LectureSchema.PAGE_TEXT_CONTENT.value, ], limit=result_limit, filters=filter_weaviate, diff --git a/app/vector_database/lecture_schema.py b/app/vector_database/lecture_schema.py index 912abed..977ae10 100644 --- a/app/vector_database/lecture_schema.py +++ b/app/vector_database/lecture_schema.py @@ -20,6 +20,7 @@ class LectureSchema(Enum): LECTURE_NAME = "lecture_name" LECTURE_UNIT_ID = "lecture_unit_id" LECTURE_UNIT_NAME = "lecture_unit_name" + LECTURE_UNIT_LINK = "lecture_unit_link" PAGE_TEXT_CONTENT = "page_text_content" PAGE_NUMBER = "page_number" BASE_URL = "base_url" @@ -78,6 +79,12 @@ def init_lecture_schema(client: WeaviateClient) -> Collection: description="The name of the lecture unit", data_type=DataType.TEXT, ), + Property( + name=LectureSchema.LECTURE_UNIT_LINK.value, + description="The link to the Lecture Unit", + data_type=DataType.TEXT, + index_searchable=False, + ), Property( name=LectureSchema.PAGE_TEXT_CONTENT.value, description="The original text content from the slide", From 1eb3c32846d3c9ca23fad348809a12ec94571750 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Thu, 11 Jul 2024 13:38:52 +0200 Subject: [PATCH 5/9] Linter --- app/pipeline/chat/lecture_chat_pipeline.py | 3 ++- app/pipeline/shared/citation_pipeline.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/pipeline/chat/lecture_chat_pipeline.py b/app/pipeline/chat/lecture_chat_pipeline.py index 297a758..12fd31c 100644 --- a/app/pipeline/chat/lecture_chat_pipeline.py +++ b/app/pipeline/chat/lecture_chat_pipeline.py @@ -46,7 +46,8 @@ def lecture_initial_prompt(): questions about the lectures. To answer them the best way, relevant lecture content is provided to you with the student's question. If the context provided to you is not enough to formulate an answer to the student question you can simply ask the student to elaborate more on his question. Use only the parts of the context provided for - you that is relevant to the student's question. If the user greets you greet him back, and ask him how you can help. + you that is relevant to the student's question. If the user greets you greet him back, + and ask him how you can help. Always formulate your answer in the same language as the user's language. """ diff --git a/app/pipeline/shared/citation_pipeline.py b/app/pipeline/shared/citation_pipeline.py index 4761b9c..7a5ac22 100644 --- a/app/pipeline/shared/citation_pipeline.py +++ b/app/pipeline/shared/citation_pipeline.py @@ -51,10 +51,10 @@ def create_formatted_string(self, paragraphs): """ formatted_string = "" for i, paragraph in enumerate(paragraphs): - lct = "Lecture: {}, Page: {}\nContent:\n---{}---\n\n".format( + lct = "Lecture: {}, Page: {}, Link: {}, \nContent:\n---{}---\n\n".format( paragraph.get(LectureSchema.LECTURE_UNIT_NAME.value), - paragraph.get(LectureSchema.LECTURE_UNIT_LINK.value), paragraph.get(LectureSchema.PAGE_NUMBER.value), + paragraph.get(LectureSchema.LECTURE_UNIT_LINK.value), paragraph.get(LectureSchema.PAGE_TEXT_CONTENT.value), ) formatted_string += lct From 2d36eecf943fdb2f8bcb6ec21aa99ec978eb3ae9 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Thu, 18 Jul 2024 02:36:53 +0200 Subject: [PATCH 6/9] Add property to the schema if it does not already exist --- app/vector_database/lecture_schema.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/app/vector_database/lecture_schema.py b/app/vector_database/lecture_schema.py index 977ae10..5308dcb 100644 --- a/app/vector_database/lecture_schema.py +++ b/app/vector_database/lecture_schema.py @@ -31,7 +31,23 @@ def init_lecture_schema(client: WeaviateClient) -> Collection: Initialize the schema for the lecture slides """ if client.collections.exists(LectureSchema.COLLECTION_NAME.value): - return client.collections.get(LectureSchema.COLLECTION_NAME.value) + collection = client.collections.get(LectureSchema.COLLECTION_NAME.value) + properties = collection.config.get(simple=True).properties + if not any( + property.name == LectureSchema.LECTURE_UNIT_LINK.value + for property_found in properties + ): + return collection + else: + collection.config.add_property( + Property( + name=LectureSchema.LECTURE_UNIT_LINK.value, + description="The link to the Lecture Unit", + data_type=DataType.TEXT, + index_searchable=False, + ) + ) + return collection return client.collections.create( name=LectureSchema.COLLECTION_NAME.value, vectorizer_config=Configure.Vectorizer.none(), From 85967cf1878f9b82f3d181aeb02aa3fbe09f6c26 Mon Sep 17 00:00:00 2001 From: Patrick Bassner Date: Tue, 27 Aug 2024 13:55:10 +0200 Subject: [PATCH 7/9] Added lecture unit to chat pipeline and citation prompts --- app/pipeline/chat/course_chat_pipeline.py | 3 ++- app/pipeline/prompts/citation_prompt.txt | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/pipeline/chat/course_chat_pipeline.py b/app/pipeline/chat/course_chat_pipeline.py index 504e448..5c00bf1 100644 --- a/app/pipeline/chat/course_chat_pipeline.py +++ b/app/pipeline/chat/course_chat_pipeline.py @@ -280,8 +280,9 @@ def lecture_content_retrieval() -> str: result = "" for paragraph in self.retrieved_paragraphs: - lct = "Lecture: {}, Page: {}\nContent:\n---{}---\n\n".format( + lct = "Lecture: {}, Unit: {}, Page: {}\nContent:\n---{}---\n\n".format( paragraph.get(LectureSchema.LECTURE_NAME.value), + paragraph.get(LectureSchema.LECTURE_UNIT_NAME.value), paragraph.get(LectureSchema.PAGE_NUMBER.value), paragraph.get(LectureSchema.PAGE_TEXT_CONTENT.value), ) diff --git a/app/pipeline/prompts/citation_prompt.txt b/app/pipeline/prompts/citation_prompt.txt index d1033b5..ba1e9e1 100644 --- a/app/pipeline/prompts/citation_prompt.txt +++ b/app/pipeline/prompts/citation_prompt.txt @@ -1,7 +1,7 @@ In the paragraphs below you are provided with an answer to a question. Underneath the answer you will find the paragraphs that the answer was based on. Add citations of the paragraphs to the answer. Cite the paragraphs in brackets after the sentence where the information is used in the answer. At the end of the answer, list each source with its corresponding number and provide the Lecture Title, page number, and a clickable link in this format: [1] "Lecture title", "page number". -If the answer uses multiple pages from the same lecture, list the page numbers at the same line separated by commas in this format : [1] "Lecture title", "page number1,number2,number3". +If the answer uses multiple pages from the same lecture, list the page numbers at the same line separated by commas in this format : [1] "Lecture title", "Lecture unit", "page number1,number2,number3". Do not Include the Actual paragraphs, only the citations at the end. Only include the citations of the paragraphs that are relevant to the answer. If the answer actually does not contain any information from the paragraphs, please do not include any citations and return '!NONE!'. @@ -11,8 +11,8 @@ Here is an example how to rewrite the answer with citations (ONLY ADD CITATION I " Lorem ipsum dolor sit amet, consectetur adipiscing elit [1]. Ded do eiusmod tempor incididunt ut labore et dolore magna aliqua [2]. -[1] Lecture 1, page 2,3,4. -[2] Lecture 2, page 5,25. +[1] Lecture 1, Unit A, page 2,3,4. +[2] Lecture 2, Unit B, page 5,25. " Here are the answer and the paragraphs: @@ -20,7 +20,7 @@ Here are the answer and the paragraphs: Answer without citations: {Answer} -Paragraphs with their Lecture Names, Links and Page Numbers: +Paragraphs with their Lecture Names, Unit Names, Links and Page Numbers: {Paragraphs} Answer with citations (ensure empty line between the message and the citations): From dc1a4cb06f67777cc05b341745ff1c4a5ffb988e Mon Sep 17 00:00:00 2001 From: Patrick Bassner Date: Tue, 27 Aug 2024 14:20:06 +0200 Subject: [PATCH 8/9] change citiation pipeline model version --- app/pipeline/shared/citation_pipeline.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/pipeline/shared/citation_pipeline.py b/app/pipeline/shared/citation_pipeline.py index 7a5ac22..4fa1a7f 100644 --- a/app/pipeline/shared/citation_pipeline.py +++ b/app/pipeline/shared/citation_pipeline.py @@ -25,7 +25,7 @@ def __init__(self): super().__init__(implementation_id="citation_pipeline") request_handler = CapabilityRequestHandler( requirements=RequirementList( - gpt_version_equivalent=3.5, + gpt_version_equivalent=4.25, context_length=16385, ) ) @@ -51,7 +51,8 @@ def create_formatted_string(self, paragraphs): """ formatted_string = "" for i, paragraph in enumerate(paragraphs): - lct = "Lecture: {}, Page: {}, Link: {}, \nContent:\n---{}---\n\n".format( + lct = "Lecture: {}, Unit: {}, Page: {}, Link: {},\nContent:\n---{}---\n\n".format( + paragraph.get(LectureSchema.LECTURE_NAME.value), paragraph.get(LectureSchema.LECTURE_UNIT_NAME.value), paragraph.get(LectureSchema.PAGE_NUMBER.value), paragraph.get(LectureSchema.LECTURE_UNIT_LINK.value), From a58c598ab75cf02eba80c2b88d1d815bfd92529f Mon Sep 17 00:00:00 2001 From: Patrick Bassner Date: Tue, 27 Aug 2024 15:29:09 +0200 Subject: [PATCH 9/9] Updated citation format instructions in citation_prompt.txt --- app/pipeline/prompts/citation_prompt.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/pipeline/prompts/citation_prompt.txt b/app/pipeline/prompts/citation_prompt.txt index ba1e9e1..f69e885 100644 --- a/app/pipeline/prompts/citation_prompt.txt +++ b/app/pipeline/prompts/citation_prompt.txt @@ -1,8 +1,8 @@ In the paragraphs below you are provided with an answer to a question. Underneath the answer you will find the paragraphs that the answer was based on. Add citations of the paragraphs to the answer. Cite the paragraphs in brackets after the sentence where the information is used in the answer. -At the end of the answer, list each source with its corresponding number and provide the Lecture Title, page number, and a clickable link in this format: [1] "Lecture title", "page number". -If the answer uses multiple pages from the same lecture, list the page numbers at the same line separated by commas in this format : [1] "Lecture title", "Lecture unit", "page number1,number2,number3". -Do not Include the Actual paragraphs, only the citations at the end. +At the end of the answer, list each source with its corresponding number and provide the Lecture Title, page number, and a clickable link in this format: [1] "Lecture title", "Lecture unit title", "page number". +If the answer uses multiple pages from the same lecture, list the page numbers at the same line separated by commas in this format : [1] "Lecture title", "Lecture unit title", "page number1,number2,number3". +Do not include the actual paragraphs, only the citations at the end. Only include the citations of the paragraphs that are relevant to the answer. If the answer actually does not contain any information from the paragraphs, please do not include any citations and return '!NONE!'. But if the answer contains information from the paragraphs, ALWAYS include citations.