diff --git a/README.md b/README.md index 53d37d848..772c15608 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ > [!Tip] > 🔔**Claude3 Opus supported.** As of 04/17/2024, Bedrock only supports the `us-west-2` region. In this repository, Bedrock uses the `us-east-1` region by default. Therefore, if you plan to use it, please change the value of `bedrockRegion` before deployment. For more details, please refer [here](#deploy-using-cdk). -> [!Info] +> [!Important] > We'd like to hear your feedback to implement bot creation permission management feature. The plan is to grant permissions to individual users through the admin panel, but this may increase operational overhead for existing users. [Please take the survey](https://github.com/aws-samples/bedrock-claude-chat/issues/161#issuecomment-2058194533). > [!Warning] diff --git a/backend/app/repositories/conversation.py b/backend/app/repositories/conversation.py index 05c5d74ba..52382d6fe 100644 --- a/backend/app/repositories/conversation.py +++ b/backend/app/repositories/conversation.py @@ -14,9 +14,11 @@ decompose_conv_id, ) from app.repositories.models.conversation import ( + ChunkModel, ContentModel, ConversationMeta, ConversationModel, + FeedbackModel, MessageModel, ) from app.utils import get_current_time @@ -204,6 +206,27 @@ def find_conversation_by_id(user_id: str, conversation_id: str) -> ConversationM children=v["children"], parent=v["parent"], create_time=float(v["create_time"]), + feedback=( + FeedbackModel( + thumbs_up=v["feedback"]["thumbs_up"], + category=v["feedback"]["category"], + comment=v["feedback"]["comment"], + ) + if v.get("feedback") + else None + ), + used_chunks=( + [ + ChunkModel( + content=c["content"], + source=c["source"], + rank=c["rank"], + ) + for c in v["used_chunks"] + ] + if v.get("used_chunks") + else None + ), ) for k, v in message_map.items() }, @@ -325,3 +348,28 @@ def change_conversation_title(user_id: str, conversation_id: str, new_title: str logger.info(f"Updated conversation title response: {response}") return response + + +def update_feedback( + user_id: str, conversation_id: str, message_id: str, feedback: FeedbackModel +): + logger.info(f"Updating feedback for conversation: {conversation_id}") + table = _get_table_client(user_id) + conv = find_conversation_by_id(user_id, conversation_id) + message_map = conv.message_map + message_map[message_id].feedback = feedback + + response = table.update_item( + Key={ + "PK": user_id, + "SK": compose_conv_id(user_id, conversation_id), + }, + UpdateExpression="set MessageMap = :m", + ExpressionAttributeValues={ + ":m": json.dumps({k: v.model_dump() for k, v in message_map.items()}) + }, + ConditionExpression="attribute_exists(PK) AND attribute_exists(SK)", + ReturnValues="UPDATED_NEW", + ) + logger.info(f"Updated feedback response: {response}") + return response diff --git a/backend/app/repositories/models/conversation.py b/backend/app/repositories/models/conversation.py index bbe30b082..656dd58e5 100644 --- a/backend/app/repositories/models/conversation.py +++ b/backend/app/repositories/models/conversation.py @@ -10,6 +10,18 @@ class ContentModel(BaseModel): body: str +class FeedbackModel(BaseModel): + thumbs_up: bool + category: str + comment: str + + +class ChunkModel(BaseModel): + content: str + source: str + rank: int + + class MessageModel(BaseModel): role: str content: list[ContentModel] @@ -17,6 +29,8 @@ class MessageModel(BaseModel): children: list[str] parent: str | None create_time: float + feedback: FeedbackModel | None + used_chunks: list[ChunkModel] | None class ConversationModel(BaseModel): diff --git a/backend/app/routes/bot.py b/backend/app/routes/bot.py index 5d2383b0a..af72d792c 100644 --- a/backend/app/routes/bot.py +++ b/backend/app/routes/bot.py @@ -27,7 +27,6 @@ remove_bot_by_id, remove_uploaded_file, ) -from app.usecases.chat import chat, fetch_conversation, propose_conversation_title from app.user import User from fastapi import APIRouter, Request diff --git a/backend/app/routes/conversation.py b/backend/app/routes/conversation.py index 1d1fb7517..18500b68e 100644 --- a/backend/app/routes/conversation.py +++ b/backend/app/routes/conversation.py @@ -3,12 +3,16 @@ delete_conversation_by_id, delete_conversation_by_user_id, find_conversation_by_user_id, + update_feedback, ) +from app.repositories.models.conversation import FeedbackModel from app.routes.schemas.conversation import ( ChatInput, ChatOutput, Conversation, ConversationMetaOutput, + FeedbackInput, + FeedbackOutput, NewTitleInput, ProposedTitle, RelatedDocumentsOutput, @@ -119,3 +123,33 @@ def get_proposed_title(request: Request, conversation_id: str): title = propose_conversation_title(current_user.id, conversation_id) return ProposedTitle(title=title) + + +@router.put( + "/conversation/{conversation_id}/{message_id}/feedback", + response_model=FeedbackOutput, +) +def put_feedback( + request: Request, + conversation_id: str, + message_id: str, + feedback_input: FeedbackInput, +): + """Send feedback.""" + current_user: User = request.state.current_user + + update_feedback( + user_id=current_user.id, + conversation_id=conversation_id, + message_id=message_id, + feedback=FeedbackModel( + thumbs_up=feedback_input.thumbs_up, + category=feedback_input.category if feedback_input.category else "", + comment=feedback_input.comment if feedback_input.comment else "", + ), + ) + return FeedbackOutput( + thumbs_up=feedback_input.thumbs_up, + category=feedback_input.category if feedback_input.category else "", + comment=feedback_input.comment if feedback_input.comment else "", + ) diff --git a/backend/app/routes/schemas/conversation.py b/backend/app/routes/schemas/conversation.py index 790e7d96b..949a1504d 100644 --- a/backend/app/routes/schemas/conversation.py +++ b/backend/app/routes/schemas/conversation.py @@ -1,7 +1,7 @@ from typing import Literal from app.routes.schemas.base import BaseSchema -from pydantic import Field +from pydantic import Field, root_validator, validator type_model_name = Literal[ "claude-instant-v1", @@ -23,6 +23,36 @@ class Content(BaseSchema): body: str = Field(..., description="Content body. Text or base64 encoded image.") +class FeedbackInput(BaseSchema): + thumbs_up: bool + category: str | None = Field( + None, description="Reason category. Required if thumbs_up is False." + ) + comment: str | None = Field(None, description="optional comment") + + @root_validator(pre=True) + def check_category(cls, values): + thumbs_up = values.get("thumbs_up") + category = values.get("category") + + if not thumbs_up and category is None: + raise ValueError("category is required if `thumbs_up` is `False`") + + return values + + +class FeedbackOutput(BaseSchema): + thumbs_up: bool + category: str + comment: str + + +class Chunk(BaseSchema): + content: str + source: str + rank: int + + class MessageInput(BaseSchema): role: str content: list[Content] @@ -38,6 +68,8 @@ class MessageOutput(BaseSchema): content: list[Content] model: type_model_name children: list[str] + feedback: FeedbackOutput | None + used_chunks: list[Chunk] | None parent: str | None diff --git a/backend/app/usecases/chat.py b/backend/app/usecases/chat.py index 175ff8c41..296024228 100644 --- a/backend/app/usecases/chat.py +++ b/backend/app/usecases/chat.py @@ -14,6 +14,7 @@ ) from app.repositories.custom_bot import find_alias_by_id, store_alias from app.repositories.models.conversation import ( + ChunkModel, ContentModel, ConversationModel, MessageModel, @@ -22,14 +23,21 @@ from app.routes.schemas.conversation import ( ChatInput, ChatOutput, + Chunk, Content, Conversation, + FeedbackOutput, MessageOutput, RelatedDocumentsOutput, ) from app.usecases.bot import fetch_bot, modify_bot_last_used_time from app.utils import get_anthropic_client, get_current_time, is_running_on_lambda -from app.vector_search import SearchResult, get_source_link, search_related_docs +from app.vector_search import ( + SearchResult, + filter_used_results, + get_source_link, + search_related_docs, +) from ulid import ULID logger = logging.getLogger(__name__) @@ -65,7 +73,7 @@ def prepare_conversation( ) initial_message_map = { - # Dummy system message + # Dummy system message, which is used for root node of the message tree. "system": MessageModel( role="system", content=[ @@ -79,6 +87,8 @@ def prepare_conversation( children=[], parent=None, create_time=current_time, + feedback=None, + used_chunks=None, ) } parent_id = "system" @@ -100,6 +110,8 @@ def prepare_conversation( children=[], parent="system", create_time=current_time, + feedback=None, + used_chunks=None, ) initial_message_map["system"].children.append("instruction") @@ -157,6 +169,8 @@ def prepare_conversation( children=[], parent=parent_id, create_time=current_time, + feedback=None, + used_chunks=None, ) conversation.message_map[message_id] = new_message conversation.message_map[parent_id].children.append(message_id) # type: ignore @@ -259,18 +273,19 @@ def chat(user_id: str, chat_input: ChatInput) -> ChatOutput: user_msg_id, conversation, bot = prepare_conversation(user_id, chat_input) message_map = conversation.message_map + search_results = [] if bot and is_running_on_lambda(): # NOTE: `is_running_on_lambda`is a workaround for local testing due to no postgres mock. # Fetch most related documents from vector store # NOTE: Currently embedding not support multi-modal. For now, use the last content. query = conversation.message_map[user_msg_id].content[-1].body - results = search_related_docs( + search_results = search_related_docs( bot_id=bot.id, limit=SEARCH_CONFIG["max_results"], query=query ) - logger.info(f"Search results from vector store: {results}") + logger.info(f"Search results from vector store: {search_results}") # Insert contexts to instruction - conversation_with_context = insert_knowledge(conversation, results) + conversation_with_context = insert_knowledge(conversation, search_results) message_map = conversation_with_context.message_map messages = trace_to_root( @@ -291,6 +306,14 @@ def chat(user_id: str, chat_input: ChatInput) -> ChatOutput: response: AnthropicMessage = client.messages.create(**args) reply_txt = response.content[0].text + # Used chunks for RAG generation + used_chunks = None + if bot and is_running_on_lambda(): + used_chunks = [ + ChunkModel(content=r.content, source=r.source, rank=r.rank) + for r in filter_used_results(reply_txt, search_results) + ] + # Issue id for new assistant message assistant_msg_id = str(ULID()) # Append bedrock output to the existing conversation @@ -301,6 +324,8 @@ def chat(user_id: str, chat_input: ChatInput) -> ChatOutput: children=[], parent=user_msg_id, create_time=get_current_time(), + feedback=None, + used_chunks=used_chunks, ) conversation.message_map[assistant_msg_id] = message @@ -341,6 +366,19 @@ def chat(user_id: str, chat_input: ChatInput) -> ChatOutput: model=message.model, children=message.children, parent=message.parent, + feedback=None, + used_chunks=( + [ + Chunk( + content=c.content, + source=c.source, + rank=c.rank, + ) + for c in message.used_chunks + ] + if message.used_chunks + else None + ), ), bot_id=conversation.bot_id, ) @@ -389,6 +427,8 @@ def propose_conversation_title( children=[], parent=conversation.last_message_id, create_time=get_current_time(), + feedback=None, + used_chunks=None, ) messages.append(new_message) @@ -419,6 +459,27 @@ def fetch_conversation(user_id: str, conversation_id: str) -> Conversation: model=message.model, children=message.children, parent=message.parent, + feedback=( + FeedbackOutput( + thumbs_up=message.feedback.thumbs_up, + category=message.feedback.category, + comment=message.feedback.comment, + ) + if message.feedback + else None + ), + used_chunks=( + [ + Chunk( + content=c.content, + source=c.source, + rank=c.rank, + ) + for c in message.used_chunks + ] + if message.used_chunks + else None + ), ) for message_id, message in conversation.message_map.items() } diff --git a/backend/app/vector_search.py b/backend/app/vector_search.py index d5d2ef493..95d87e302 100644 --- a/backend/app/vector_search.py +++ b/backend/app/vector_search.py @@ -1,6 +1,7 @@ import json import logging import os +import re from typing import Literal import pg8000 @@ -24,6 +25,31 @@ class SearchResult(BaseModel): rank: int +def filter_used_results( + generated_text: str, search_results: list[SearchResult] +) -> list[SearchResult]: + """Filter the search results based on the citations in the generated text. + Note that the citations in the generated text are in the format of [^rank]. + """ + used_results: list[SearchResult] = [] + + try: + # Extract citations from the generated text + citations = [ + citation.strip("[]^") + for citation in re.findall(r"\[\^(\d+)\]", generated_text) + ] + except Exception as e: + logger.error(f"Error extracting citations from the generated text: {e}") + return used_results + + for result in search_results: + if str(result.rank) in citations: + used_results.append(result) + + return used_results + + def get_source_link(source: str) -> tuple[Literal["s3", "url"], str]: if source.startswith("s3://"): s3_path = source[5:] # Remove "s3://" prefix diff --git a/backend/app/websocket.py b/backend/app/websocket.py index 1f917a28e..e6e0144e0 100644 --- a/backend/app/websocket.py +++ b/backend/app/websocket.py @@ -11,12 +11,12 @@ from app.bedrock import calculate_price, compose_args_for_anthropic_client from app.config import GENERATION_CONFIG, SEARCH_CONFIG from app.repositories.conversation import RecordNotFoundError, store_conversation -from app.repositories.models.conversation import ContentModel, MessageModel +from app.repositories.models.conversation import ChunkModel, ContentModel, MessageModel from app.routes.schemas.conversation import ChatInputWithToken from app.usecases.bot import modify_bot_last_used_time from app.usecases.chat import insert_knowledge, prepare_conversation, trace_to_root from app.utils import get_anthropic_client, get_current_time -from app.vector_search import search_related_docs +from app.vector_search import filter_used_results, search_related_docs from boto3.dynamodb.conditions import Key from ulid import ULID @@ -63,6 +63,7 @@ def process_chat_input( return {"statusCode": 400, "body": "Invalid request."} message_map = conversation.message_map + search_results = [] if bot and bot.has_knowledge(): gatewayapi.post_to_connection( ConnectionId=connection_id, @@ -75,13 +76,13 @@ def process_chat_input( # Fetch most related documents from vector store # NOTE: Currently embedding not support multi-modal. For now, use the last text content. query = conversation.message_map[user_msg_id].content[-1].body - results = search_related_docs( + search_results = search_related_docs( bot_id=bot.id, limit=SEARCH_CONFIG["max_results"], query=query ) - logger.info(f"Search results from vector store: {results}") + logger.info(f"Search results from vector store: {search_results}") # Insert contexts to instruction - conversation_with_context = insert_knowledge(conversation, results) + conversation_with_context = insert_knowledge(conversation, search_results) message_map = conversation_with_context.message_map messages = trace_to_root( @@ -151,6 +152,15 @@ def process_chat_input( elif isinstance(event, MessageStopEvent): # Persist conversation before finish streaming so that front-end can avoid 404 issue concatenated = "".join(completions) + + # Used chunks for RAG generation + used_chunks = None + if bot: + used_chunks = [ + ChunkModel(content=r.content, source=r.source, rank=r.rank) + for r in filter_used_results(concatenated, search_results) + ] + # Append entire completion as the last message assistant_msg_id = str(ULID()) message = MessageModel( @@ -164,6 +174,8 @@ def process_chat_input( children=[], parent=user_msg_id, create_time=get_current_time(), + feedback=None, + used_chunks=used_chunks, ) conversation.message_map[assistant_msg_id] = message # Append children to parent diff --git a/backend/tests/test_repositories/test_conversation.py b/backend/tests/test_repositories/test_conversation.py index f9b98a675..faa177aff 100644 --- a/backend/tests/test_repositories/test_conversation.py +++ b/backend/tests/test_repositories/test_conversation.py @@ -1,31 +1,28 @@ import sys import unittest - sys.path.append(".") from app.config import DEFAULT_EMBEDDING_CONFIG - from app.repositories.conversation import ( ContentModel, ConversationModel, MessageModel, RecordNotFoundError, - _get_table_client, change_conversation_title, - compose_conv_id, delete_conversation_by_id, delete_conversation_by_user_id, find_conversation_by_id, find_conversation_by_user_id, store_conversation, + update_feedback, ) from app.repositories.custom_bot import ( delete_bot_by_id, - find_private_bot_by_id, find_private_bots_by_user_id, store_bot, ) +from app.repositories.models.conversation import FeedbackModel from app.repositories.models.custom_bot import ( BotModel, EmbeddingParamsModel, @@ -138,6 +135,8 @@ def test_store_and_find_conversation(self): children=["x", "y"], parent="z", create_time=1627984879.9, + feedback=None, + used_chunks=None, ) }, last_message_id="x", @@ -189,6 +188,25 @@ def test_store_and_find_conversation(self): ) self.assertEqual(found_conversation.title, "Updated title") + # Test give a feedback + self.assertIsNone(found_conversation.message_map["a"].feedback) + response = update_feedback( + user_id="user", + conversation_id="1", + message_id="a", + feedback=FeedbackModel( + thumbs_up=True, category="Good", comment="The response is pretty good." + ), + ) + found_conversation = find_conversation_by_id( + user_id="user", conversation_id="1" + ) + feedback = found_conversation.message_map["a"].feedback + self.assertIsNotNone(feedback) + self.assertEqual(feedback.thumbs_up, True) # type: ignore + self.assertEqual(feedback.category, "Good") # type: ignore + self.assertEqual(feedback.comment, "The response is pretty good.") # type: ignore + # Test deleting conversation by id delete_conversation_by_id(user_id="user", conversation_id="1") with self.assertRaises(RecordNotFoundError): @@ -217,6 +235,8 @@ def test_store_and_find_large_conversation(self): children=[], parent=None, create_time=1627984879.9, + feedback=None, + used_chunks=None, ) for i in range(10) # Create 10 large messages } @@ -297,6 +317,8 @@ def setUp(self) -> None: children=["x", "y"], parent="z", create_time=1627984879.9, + feedback=None, + used_chunks=None, ) }, last_message_id="x", @@ -324,6 +346,8 @@ def setUp(self) -> None: children=["x", "y"], parent="z", create_time=1627984879.9, + feedback=None, + used_chunks=None, ) }, last_message_id="x", diff --git a/backend/tests/test_routes/test_schemas/test_conversation.py b/backend/tests/test_routes/test_schemas/test_conversation.py new file mode 100644 index 000000000..c18f6d082 --- /dev/null +++ b/backend/tests/test_routes/test_schemas/test_conversation.py @@ -0,0 +1,29 @@ +import sys + +sys.path.append(".") +import unittest + +from app.routes.schemas.conversation import FeedbackInput +from pydantic import ValidationError + + +class TestFeedbackInput(unittest.TestCase): + def test_create_input_valid_no_category(self): + obj = FeedbackInput(thumbs_up=True, category=None, comment="Excellent!") + self.assertTrue(obj.thumbs_up) + self.assertIsNone(obj.category) + self.assertEqual(obj.comment, "Excellent!") + + def test_create_input_invalid_no_category(self): + with self.assertRaises(ValidationError): + FeedbackInput(thumbs_up=False, category=None, comment="Needs improvement.") + + def test_create_input_valid_no_comment(self): + obj = FeedbackInput(thumbs_up=False, category="DISLIKE", comment=None) + self.assertFalse(obj.thumbs_up) + self.assertEqual(obj.category, "DISLIKE") + self.assertIsNone(obj.comment) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/tests/test_usecases/test_chat.py b/backend/tests/test_usecases/test_chat.py index 809fba2b4..5d829099f 100644 --- a/backend/tests/test_usecases/test_chat.py +++ b/backend/tests/test_usecases/test_chat.py @@ -4,12 +4,6 @@ import unittest from pprint import pprint -from tests.test_usecases.utils.bot_factory import ( - create_test_private_bot, - create_test_public_bot, - create_test_instruction_template, -) - from anthropic.types import MessageStopEvent from app.bedrock import get_model_id from app.config import GENERATION_CONFIG @@ -30,7 +24,6 @@ ConversationModel, MessageModel, ) - from app.routes.schemas.conversation import ( ChatInput, ChatOutput, @@ -48,6 +41,11 @@ ) from app.utils import get_anthropic_client from app.vector_search import SearchResult +from tests.test_usecases.utils.bot_factory import ( + create_test_instruction_template, + create_test_private_bot, + create_test_public_bot, +) MODEL: type_model_name = "claude-instant-v1" @@ -64,6 +62,8 @@ def test_trace_to_root(self): children=["bot_1"], parent=None, create_time=1627984879.9, + feedback=None, + used_chunks=None, ), "bot_1": MessageModel( role="assistant", @@ -74,6 +74,8 @@ def test_trace_to_root(self): children=["user_2"], parent="user_1", create_time=1627984879.9, + feedback=None, + used_chunks=None, ), "user_2": MessageModel( role="user", @@ -84,6 +86,8 @@ def test_trace_to_root(self): children=["bot_2"], parent="bot_1", create_time=1627984879.9, + feedback=None, + used_chunks=None, ), "bot_2": MessageModel( role="assistant", @@ -94,6 +98,8 @@ def test_trace_to_root(self): children=["user_3a", "user_3b"], parent="user_2", create_time=1627984879.9, + feedback=None, + used_chunks=None, ), "user_3a": MessageModel( role="user", @@ -104,6 +110,8 @@ def test_trace_to_root(self): children=[], parent="bot_2", create_time=1627984879.9, + feedback=None, + used_chunks=None, ), "user_3b": MessageModel( role="user", @@ -114,6 +122,8 @@ def test_trace_to_root(self): children=[], parent="bot_2", create_time=1627984879.9, + feedback=None, + used_chunks=None, ), } messages = trace_to_root("user_3a", message_map) @@ -241,6 +251,8 @@ def setUp(self) -> None: children=["1-assistant"], parent=None, create_time=1627984879.9, + feedback=None, + used_chunks=None, ), "1-assistant": MessageModel( role="assistant", @@ -255,6 +267,8 @@ def setUp(self) -> None: children=[], parent="1-user", create_time=1627984879.9, + feedback=None, + used_chunks=None, ), }, bot_id=None, @@ -325,6 +339,8 @@ def setUp(self) -> None: children=["a-2"], parent=None, create_time=1627984879.9, + feedback=None, + used_chunks=None, ), "a-2": MessageModel( role="assistant", @@ -339,6 +355,8 @@ def setUp(self) -> None: children=[], parent="a-1", create_time=1627984879.9, + feedback=None, + used_chunks=None, ), "b-1": MessageModel( role="user", @@ -353,6 +371,8 @@ def setUp(self) -> None: children=["b-2"], parent=None, create_time=1627984879.9, + feedback=None, + used_chunks=None, ), "b-2": MessageModel( role="assistant", @@ -367,6 +387,8 @@ def setUp(self) -> None: children=[], parent="b-1", create_time=1627984879.9, + feedback=None, + used_chunks=None, ), }, bot_id=None, @@ -687,6 +709,8 @@ def test_insert_knowledge(self): children=["1-user"], parent=None, create_time=1627984879.9, + feedback=None, + used_chunks=None, ), "1-user": MessageModel( role="user", @@ -701,6 +725,8 @@ def test_insert_knowledge(self): children=[], parent="instruction", create_time=1627984879.9, + feedback=None, + used_chunks=None, ), }, bot_id="bot1", diff --git a/backend/tests/test_vector_search/test_vector_search.py b/backend/tests/test_vector_search/test_vector_search.py new file mode 100644 index 000000000..a6fa52ada --- /dev/null +++ b/backend/tests/test_vector_search/test_vector_search.py @@ -0,0 +1,52 @@ +import sys +import unittest + +sys.path.append(".") + +from app.vector_search import SearchResult, filter_used_results + + +class TestVectorSearch(unittest.TestCase): + def test_filter_used_results(self): + search_results = [ + SearchResult(bot_id="1", content="content1", source="source1", rank=1), + SearchResult(bot_id="2", content="content2", source="source2", rank=2), + SearchResult(bot_id="3", content="content3", source="source3", rank=3), + ] + + generated_text = "This is a test [^1] [^3]" + + used_results = filter_used_results(generated_text, search_results) + self.assertEqual(len(used_results), 2) + self.assertEqual(used_results[0].rank, 1) + self.assertEqual(used_results[1].rank, 3) + + def test_no_reference_filter_used_results(self): + search_results = [ + SearchResult(bot_id="1", content="content1", source="source1", rank=1), + SearchResult(bot_id="2", content="content2", source="source2", rank=2), + SearchResult(bot_id="3", content="content3", source="source3", rank=3), + ] + + # 4 is not in the search results + generated_text = "This is a test [^4]" + + used_results = filter_used_results(generated_text, search_results) + self.assertEqual(len(used_results), 0) + + def test_format_not_match_filter_used_results(self): + search_results = [ + SearchResult(bot_id="1", content="content1", source="source1", rank=1), + SearchResult(bot_id="2", content="content2", source="source2", rank=2), + SearchResult(bot_id="3", content="content3", source="source3", rank=3), + ] + + # format not match + generated_text = "This is a test 1 3" + + used_results = filter_used_results(generated_text, search_results) + self.assertEqual(len(used_results), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/cdk/lib/constructs/usage-analysis.ts b/cdk/lib/constructs/usage-analysis.ts index 493e3a301..985f85dd2 100644 --- a/cdk/lib/constructs/usage-analysis.ts +++ b/cdk/lib/constructs/usage-analysis.ts @@ -80,6 +80,10 @@ export class UsageAnalysis extends Construct { name: "MessageMap", type: glue.Schema.struct([{ name: "S", type: glue.Schema.STRING }]), }, + { + name: "IsLargeMessage", + type: glue.Schema.struct([{ name: "BOOL", type: glue.Schema.BOOLEAN }]), + }, { name: "PK", type: glue.Schema.struct([{ name: "S", type: glue.Schema.STRING }]), diff --git a/docs/ADMINISTRATOR.md b/docs/ADMINISTRATOR.md index 2784a9089..aa82060d3 100644 --- a/docs/ADMINISTRATOR.md +++ b/docs/ADMINISTRATOR.md @@ -2,6 +2,14 @@ The administrator dashboard is a vital tool as it provides essential insights into custom bot usage and user behavior. Without the functionality, it would be challenging for administrators to understand which custom bots are popular, why they are popular, and who is using them. This information is crucial for optimizing instruction prompts, customizing RAG data sources, and identifying heavy users who might will be an influencer. +### Feedback loop + +The output from LLM may not always meet the user's expectations. Sometimes it fails to satisfy the user's needs. To effectively "integrate" LLMs into business operations and daily life, implementing a feedback loop is essential. Bedrock Claude Chat is equipped with a feedback feature designed to enable users to analyze why dissatisfaction arose. Based on the analysis results, users can adjust the prompts, RAG data sources, and parameters accordingly. + +![](./imgs/feedback_loop.png) + +![](./imgs/feedback.png) + ## Features Currently provides a basic overview of chatbot and user usage, focusing on aggregating data for each bot and user over specified time periods and sorting the results by usage fees. @@ -27,7 +35,7 @@ The admin user must be a member of group called `Admin`, which can be set up via ## Download conversation data -You can query the conversation logs by Athena, using SQL. To download logs, open Athena Query Editor from management console and run SQL. Followings are some example queries which are useful to analyze use-cases. +You can query the conversation logs by Athena, using SQL. To download logs, open Athena Query Editor from management console and run SQL. Followings are some example queries which are useful to analyze use-cases. Feedback can be referred in `MessageMap` attribute. ### Query per Bot ID diff --git a/docs/README_ja.md b/docs/README_ja.md index 6459667b0..9823c8af1 100644 --- a/docs/README_ja.md +++ b/docs/README_ja.md @@ -1,11 +1,9 @@ # Bedrock Claude Chat -![](https://github.com/aws-samples/bedrock-claude-chat/actions/workflows/test.yml/badge.svg) - > [!Tip] > 🔔**Claude3 Opus をサポートしました。** 2024/04/17 現在、Bedrock は`us-west-2`のみサポートしています。このリポジトリでは Bedrock はデフォルトで`us-east-1`リージョンを利用します。このため、ご利用される場合はデプロイ前に`bedrockRegion`の値を変更してください。詳細は[こちら](#deploy-using-cdk) -> [!Info] +> [!Important] > ボット作成権限の管理機能実装について、ご意見をお聞かせください。管理者パネルから個別にユーザーに権限を付与する予定ですが、既存ユーザーの皆さんの運用オーバーヘッドが大幅に増加する可能性があります。[アンケートにご協力ください](https://github.com/aws-samples/bedrock-claude-chat/issues/161#issuecomment-2058194533)。 > [!Warning] diff --git a/docs/imgs/feedback.png b/docs/imgs/feedback.png new file mode 100644 index 000000000..3478ae760 Binary files /dev/null and b/docs/imgs/feedback.png differ diff --git a/docs/imgs/feedback_loop.png b/docs/imgs/feedback_loop.png new file mode 100644 index 000000000..3d0caa78d Binary files /dev/null and b/docs/imgs/feedback_loop.png differ diff --git a/frontend/src/@types/conversation.d.ts b/frontend/src/@types/conversation.d.ts index b66fef344..1ff3b5ee9 100644 --- a/frontend/src/@types/conversation.d.ts +++ b/frontend/src/@types/conversation.d.ts @@ -15,6 +15,7 @@ export type MessageContent = { role: Role; content: Content[]; model: Model; + feedback: null | Feedback; }; export type RelatedDocument = { @@ -74,3 +75,15 @@ export type MessageMap = { export type Conversation = ConversationMeta & { messageMap: MessageMap; }; + +export type PutFeedbackRequest = { + thumbsUp: boolean; + category: null | string; + comment: null | string; +}; + +export type Feedback = { + thumbsUp: boolean; + category: string; + comment: string; +}; diff --git a/frontend/src/components/ChatMessage.tsx b/frontend/src/components/ChatMessage.tsx index 504c04648..168adcd94 100644 --- a/frontend/src/components/ChatMessage.tsx +++ b/frontend/src/components/ChatMessage.tsx @@ -1,15 +1,26 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react'; import ChatMessageMarkdown from './ChatMessageMarkdown'; import ButtonCopy from './ButtonCopy'; -import { PiCaretLeftBold, PiNotePencil, PiUserFill } from 'react-icons/pi'; +import { + PiCaretLeftBold, + PiNotePencil, + PiUserFill, + PiThumbsDown, + PiThumbsDownFill, +} from 'react-icons/pi'; import { BaseProps } from '../@types/common'; -import { DisplayMessageContent, RelatedDocument } from '../@types/conversation'; +import { + DisplayMessageContent, + RelatedDocument, + PutFeedbackRequest, +} from '../@types/conversation'; import ButtonIcon from './ButtonIcon'; import Textarea from './Textarea'; import Button from './Button'; import ModalDialog from './ModalDialog'; import { useTranslation } from 'react-i18next'; import useChat from '../hooks/useChat'; +import DialogFeedback from './DialogFeedback'; type Props = BaseProps & { chatContent?: DisplayMessageContent; @@ -21,8 +32,9 @@ const ChatMessage: React.FC = (props) => { const { t } = useTranslation(); const [isEdit, setIsEdit] = useState(false); const [changedContent, setChangedContent] = useState(''); + const [isFeedbackOpen, setIsFeedbackOpen] = useState(false); - const { getRelatedDocuments } = useChat(); + const { getRelatedDocuments, conversationId, giveFeedback } = useChat(); const [relatedDocuments, setRelatedDocuments] = useState( [] ); @@ -61,6 +73,16 @@ const ChatMessage: React.FC = (props) => { setIsEdit(false); }, [changedContent, chatContent?.sibling, props]); + const handleFeedbackSubmit = useCallback( + (messageId: string, feedback: PutFeedbackRequest) => { + if (chatContent && conversationId) { + giveFeedback(messageId, feedback); + } + setIsFeedbackOpen(false); + }, + [chatContent, conversationId, giveFeedback] + ); + return (
@@ -174,10 +196,10 @@ const ChatMessage: React.FC = (props) => {
-
+
{chatContent?.role === 'user' && !isEdit && ( { setChangedContent(chatContent.content[0].body); setIsEdit(true); @@ -186,15 +208,35 @@ const ChatMessage: React.FC = (props) => { )} {chatContent?.role === 'assistant' && ( - <> +
+ setIsFeedbackOpen(true)}> + {chatContent.feedback && !chatContent.feedback.thumbsUp ? ( + + ) : ( + + )} + - +
)}
+ setIsFeedbackOpen(false)} + onSubmit={(feedback) => { + if (chatContent) { + handleFeedbackSubmit(chatContent.id, feedback); + } + }} + />
); }; diff --git a/frontend/src/components/DialogFeedback.tsx b/frontend/src/components/DialogFeedback.tsx new file mode 100644 index 000000000..e8ae23cd4 --- /dev/null +++ b/frontend/src/components/DialogFeedback.tsx @@ -0,0 +1,66 @@ +import React, { useState } from 'react'; +import { BaseProps } from '../@types/common'; +import Button from './Button'; +import ModalDialog from './ModalDialog'; +import { useTranslation } from 'react-i18next'; +import Textarea from './Textarea'; +import Select from './Select'; +import { Feedback } from '../@types/conversation'; + +type Props = BaseProps & { + isOpen: boolean; + thumbsUp: boolean; + feedback?: Feedback; + onSubmit: (feedback: Feedback) => void; + onClose: () => void; +}; + +const DialogFeedback: React.FC = (props) => { + const { t } = useTranslation(); + const categoryOptions = t('feedbackDialog.categories', { + returnObjects: true, + }); + const [category, setCategory] = useState( + props.feedback?.category || categoryOptions[0].value + ); + const [comment, setComment] = useState(props.feedback?.comment || ''); + + const handleSubmit = () => { + props.onSubmit({ thumbsUp: props.thumbsUp, category, comment }); + }; + + return ( + +
+
{t('feedbackDialog.content')}
+ +