Skip to content

Commit

Permalink
tests[patch]: update API ref for chat models (#28594)
Browse files Browse the repository at this point in the history
  • Loading branch information
ccurme authored Dec 7, 2024
1 parent 0eb7ab6 commit 80a88f8
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 35 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,9 @@
"\n",
"Tests for \"optional\" capabilities are controlled via a set of properties that can be overridden on the test model subclass.\n",
"\n",
"You can see the entire list of properties in the API reference [here](https://python.langchain.com/api_reference/standard_tests/unit_tests/langchain_tests.unit_tests.chat_models.ChatModelTests.html). These properties are shared by both unit and integration tests.\n",
"You can see the entire list of properties in the API references for\n",
"[unit tests](https://python.langchain.com/api_reference/standard_tests/unit_tests/langchain_tests.unit_tests.chat_models.ChatModelUnitTests.html)\n",
"and [integration tests](https://python.langchain.com/api_reference/standard_tests/integration_tests/langchain_tests.integration_tests.chat_models.ChatModelIntegrationTests.html).\n",
"\n",
"For example, to enable integration tests for image inputs, we can implement\n",
"\n",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,14 @@ def chat_model_params(self) -> dict:
By default, this is determined by whether the chat model's `bind_tools` method
is overridden. It typically does not need to be overridden on the test class.
Example override:
.. code-block:: python
@property
def has_tool_calling(self) -> bool:
return True
.. dropdown:: tool_choice_value
Value to use for tool choice when used in tests.
Expand Down Expand Up @@ -308,7 +316,7 @@ def supports_image_tool_message(self) -> bool:

@property
def standard_chat_model_params(self) -> dict:
""":meta private:"""
""":private:"""
return {}

def test_invoke(self, model: BaseChatModel) -> None:
Expand Down Expand Up @@ -1323,8 +1331,8 @@ def has_tool_calling(self) -> bool:
.. code-block:: python
@pytest.mark.xfail(reason=("Not implemented."))
def test_tool_message_histories_string_content(self, model: BaseChatModel) -> None:
super().test_tool_message_histories_string_content(model)
def test_tool_message_histories_string_content(self, *args: Any) -> None:
super().test_tool_message_histories_string_content(*args)
""" # noqa: E501
if not self.has_tool_calling:
pytest.skip("Test requires tool calling.")
Expand Down Expand Up @@ -1407,8 +1415,8 @@ def has_tool_calling(self) -> bool:
.. code-block:: python
@pytest.mark.xfail(reason=("Not implemented."))
def test_tool_message_histories_list_content(self, model: BaseChatModel) -> None:
super().test_tool_message_histories_list_content(model)
def test_tool_message_histories_list_content(self, *args: Any) -> None:
super().test_tool_message_histories_list_content(*args)
""" # noqa: E501
if not self.has_tool_calling:
pytest.skip("Test requires tool calling.")
Expand Down Expand Up @@ -1489,8 +1497,8 @@ def has_tool_calling(self) -> bool:
.. code-block:: python
@pytest.mark.xfail(reason=("Not implemented."))
def test_structured_few_shot_examples(self, model: BaseChatModel) -> None:
super().test_structured_few_shot_examples(model)
def test_structured_few_shot_examples(self, *args: Any) -> None:
super().test_structured_few_shot_examples(*args)
""" # noqa: E501
if not self.has_tool_calling:
pytest.skip("Test requires tool calling.")
Expand Down Expand Up @@ -1852,16 +1860,21 @@ def test_message_with_name(self, model: BaseChatModel) -> None:
assert len(result.content) > 0

def invoke_with_audio_input(self, *, stream: bool = False) -> AIMessage:
""":private:"""
raise NotImplementedError()

def invoke_with_audio_output(self, *, stream: bool = False) -> AIMessage:
""":private:"""
raise NotImplementedError()

def invoke_with_reasoning_output(self, *, stream: bool = False) -> AIMessage:
""":private:"""
raise NotImplementedError()

def invoke_with_cache_read_input(self, *, stream: bool = False) -> AIMessage:
""":private:"""
raise NotImplementedError()

def invoke_with_cache_creation_input(self, *, stream: bool = False) -> AIMessage:
""":private:"""
raise NotImplementedError()
56 changes: 29 additions & 27 deletions libs/standard-tests/langchain_tests/unit_tests/chat_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,17 +77,17 @@ class ChatModelTests(BaseStandardTests):
@property
@abstractmethod
def chat_model_class(self) -> Type[BaseChatModel]:
"""The chat model class to test, e.g., `ChatParrotLink`."""
"""The chat model class to test, e.g., ``ChatParrotLink``."""
...

@property
def chat_model_params(self) -> dict:
"""Initialization parameters for the chat mobdel."""
"""Initialization parameters for the chat model."""
return {}

@property
def standard_chat_model_params(self) -> dict:
""":meta private:"""
""":private:"""
return {
"temperature": 0,
"max_tokens": 100,
Expand All @@ -98,8 +98,7 @@ def standard_chat_model_params(self) -> dict:

@pytest.fixture
def model(self) -> BaseChatModel:
"""Fixture that returns an instance of the chat model. Should not be
overridden."""
""":private:"""
return self.chat_model_class(
**{**self.standard_chat_model_params, **self.chat_model_params}
)
Expand All @@ -115,51 +114,49 @@ def my_adder_tool(a: int, b: int) -> int:

@property
def has_tool_calling(self) -> bool:
"""Boolean property indicating whether the model supports tool calling."""
"""(bool) whether the model supports tool calling."""
return self.chat_model_class.bind_tools is not BaseChatModel.bind_tools

@property
def tool_choice_value(self) -> Optional[str]:
"""Value to use for tool choice when used in tests."""
"""(None or str) to use for tool choice when used in tests."""
return None

@property
def has_structured_output(self) -> bool:
"""Boolean property indicating whether the chat model supports structured
output."""
"""(bool) whether the chat model supports structured output."""
return (
self.chat_model_class.with_structured_output
is not BaseChatModel.with_structured_output
)

@property
def supports_image_inputs(self) -> bool:
"""Boolean property indicating whether the chat model supports image inputs.
Defaults to ``False``."""
"""(bool) whether the chat model supports image inputs, defaults to
``False``."""
return False

@property
def supports_video_inputs(self) -> bool:
"""Boolean property indicating whether the chat model supports image inputs.
Defaults to ``False``. No current tests are written for this feature."""
"""(bool) whether the chat model supports video inputs, efaults to ``False``.
No current tests are written for this feature."""
return False

@property
def returns_usage_metadata(self) -> bool:
"""Boolean property indicating whether the chat model returns usage metadata
on invoke and streaming responses."""
"""(bool) whether the chat model returns usage metadata on invoke and streaming
responses."""
return True

@property
def supports_anthropic_inputs(self) -> bool:
"""Boolean property indicating whether the chat model supports Anthropic-style
inputs."""
"""(bool) whether the chat model supports Anthropic-style inputs."""
return False

@property
def supports_image_tool_message(self) -> bool:
"""Boolean property indicating whether the chat model supports ToolMessages
that include image content."""
"""(bool) whether the chat model supports ToolMessages that include image
content."""
return False

@property
Expand All @@ -177,9 +174,8 @@ def supported_usage_metadata_details(
]
],
]:
"""Property controlling what usage metadata details are emitted in both invoke
and stream. Only needs to be overridden if these details are returned by the
model."""
"""(dict) what usage metadata details are emitted in invoke and stream. Only
needs to be overridden if these details are returned by the model."""
return {"invoke": [], "stream": []}


Expand Down Expand Up @@ -250,6 +246,14 @@ def chat_model_params(self) -> dict:
By default, this is determined by whether the chat model's `bind_tools` method
is overridden. It typically does not need to be overridden on the test class.
Example override:
.. code-block:: python
@property
def has_tool_calling(self) -> bool:
return True
.. dropdown:: tool_choice_value
Value to use for tool choice when used in tests.
Expand Down Expand Up @@ -445,17 +449,15 @@ def init_from_env_params(self) -> Tuple[dict, dict, dict]:

@property
def standard_chat_model_params(self) -> dict:
""":meta private:"""
""":private:"""
params = super().standard_chat_model_params
params["api_key"] = "test"
return params

@property
def init_from_env_params(self) -> Tuple[dict, dict, dict]:
"""This property is used in unit tests to test initialization from environment
variables. It should return a tuple of three dictionaries that specify the
environment variables, additional initialization args, and expected instance
attributes to check."""
"""(tuple) environment variables, additional initialization args, and expected
instance attributes for testing initialization from environment variables."""
return {}, {}, {}

def test_init(self) -> None:
Expand Down

0 comments on commit 80a88f8

Please sign in to comment.