@@ -115,6 +139,7 @@ The following table shows the keys that should be returned from the `format_inst
| source_idx |
int |
+ Index of correct context (winogrande) |
| options |
@@ -139,11 +164,28 @@ The following table shows the keys that should be returned from the `format_inst
int |
- | No need to return from format_instance |
+ No need to return from format_instance |
+ num_turns |
+ int |
+ Total turns of multi-turn conversation (default: 0) |
+
+
+ | turn_idx |
+ int |
+ evaluation_data: 0..num_turns-1 (inclusive)
example_data: -1 |
+
+
+ | real_num_shots |
+ int |
+
+ The num shots formatted as examples
+ |
+
+
| example_idx |
int |
- The index of examples if greater or equal than 0. Equal to -1 if it is not formatting an example
+ evaluation_data: -1
example_data: 0..real_num_shots-1 (inclusive)
|
@@ -165,13 +207,6 @@ The following table shows the keys that should be returned from the `format_inst
Display name of current dataset
-
- | real_num_shots |
- int |
-
- The num shots formatted as examples
- |
-
@@ -244,7 +279,7 @@ And in `get_prob` method, this will generate one evaluation instance:
]
```
-### Formating Multiple-Context Dataset:
+### Formatting Multiple-Context Dataset:
For datasets with multiple contexts (like `winogrande`), you can format the instance as follows:
@@ -270,7 +305,7 @@ which generates two evaluation instances:
]
```
-### Formating Generation Dataset:
+### Formatting Generation Dataset:
```python
INSTRUCTION = """Answer the following question.
@@ -300,7 +335,30 @@ class MyDataset(GenerationDataset):
To evaluate a pre-trained model that lacks instruction-following capabilities, you can provide an instruction explicitly by assigning a completion instruction `NO_INSTRUCTION = "{question}"` to the model, or pass an argument `--instruction "{question}"` in the command line.
-## Formating the references
+### Formatting Multi-Turn Dataset:
+
+For multi-turn datasets (like `mt_bench`), you can format the instance as follows:
+
+```python
+class MyDataset(GenerationDataset):
+
+ instruction = "{{source[turn_idx]}}"
+ metrics = [GPTEval(multi_turn=False)]
+ multi_turn = True
+ ...
+
+ def format_instance(self, instance):
+ return dict(
+ source=[instance["question_1"], instance["question_2"]]
+ )
+
+ def post_processing(self, predictions: List[Tuple[str, ...]]):
+ # By default, model outputs a tuple of predictions for each turn
+ # You can implement the post_processing method to return the last turn's assistant output
+ return [pred[-1] for pred in predictions]
+```
+
+## Formatting the references
You can implement the `references` method to return the reference answers for evaluation.
diff --git a/docs/utilization/how-to-customize-model.md b/docs/utilization/how-to-customize-model.md
index b516d1d7..6f638fb3 100644
--- a/docs/utilization/how-to-customize-model.md
+++ b/docs/utilization/how-to-customize-model.md
@@ -25,4 +25,12 @@ class NewModel(Model):
return results
```
-And then, you should register your model in the [`load`](https://github.com/RUCAIBox/LLMBox/tree/main/utilization/model/load.py) file.
+And then, you should register your model with `register_model` decorator:
+
+```python
+@register_model(model_backend="new_provider")
+def load_new_model(args: "ModelArguments"):
+ logger.info(f"Loading OpenAI API model `{args.model_name_or_path}`.")
+
+ return NewModel(args)
+```
diff --git a/tests/requirements-tests.txt b/tests/requirements-tests.txt
index 18438e04..a9fd23f6 100644
--- a/tests/requirements-tests.txt
+++ b/tests/requirements-tests.txt
@@ -37,3 +37,4 @@ language_data
google-api-python-client
immutabledict
langdetect
+scipy
diff --git a/tests/utilization/dataset/test_formatting.py b/tests/utilization/dataset/test_formatting.py
index decd71c5..8e173ba2 100644
--- a/tests/utilization/dataset/test_formatting.py
+++ b/tests/utilization/dataset/test_formatting.py
@@ -385,3 +385,41 @@ def test_formatting(get_dataset, dataset_name, num_shots, cot, ranking_type):
formatted_instance = "".join(p for p in formatted_instance if isinstance(p, str))
assert formatted_instance == eval_instance
+
+
+def test_multi_turn(get_dataset):
+ dataset = get_dataset(dataset_name="mt_bench", num_shots=0, ranking_type="generation")
+
+ assert dataset.evaluation_instances[0].messages == [{
+ "content":
+ "Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions.",
+ "role": "user",
+ }]
+ dataset.evaluation_instances[0].add_multi_turn(assistant="The fake model response.")
+ assert dataset.evaluation_instances[0].messages == [{
+ "content":
+ "Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions.",
+ "role": "user",
+ }, {
+ "content": "The fake model response.",
+ "role": "assistant",
+ }, {
+ "content": "Rewrite your previous response. Start every sentence with the letter A.",
+ "role": "user",
+ }]
+
+ dataset.evaluation_instances[0].add_multi_turn(assistant="A fake model response.")
+ assert dataset.evaluation_instances[0].messages == [{
+ "content":
+ "Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions.",
+ "role": "user",
+ }, {
+ "content": "The fake model response.",
+ "role": "assistant",
+ }, {
+ "content": "Rewrite your previous response. Start every sentence with the letter A.",
+ "role": "user",
+ }, {
+ "content": "A fake model response.",
+ "role": "assistant",
+ }]
diff --git a/tests/utilization/model/test_apply_prompt_template.py b/tests/utilization/model/test_apply_prompt_template.py
index 00d3d708..77ee3170 100644
--- a/tests/utilization/model/test_apply_prompt_template.py
+++ b/tests/utilization/model/test_apply_prompt_template.py
@@ -39,7 +39,7 @@ def test_no_smart_space(conversation: Conversation):
"assistant_start": "",
"assistant_end": "",
"auto_leading_space": False,
- "default_stops": [],
+ "default_stop": [],
}
formatter = ConversationFormatter(prompt_config, DEFAULT_CHAT_TEMPLATE)
conversation.set_formatter(formatter)
@@ -58,7 +58,7 @@ def test_smart_space(conversation: Conversation):
"assistant_start": "",
"assistant_end": "",
"auto_leading_space": True,
- "default_stops": [],
+ "default_stop": [],
}
formatter = ConversationFormatter(prompt_config, DEFAULT_CHAT_TEMPLATE)
conversation[2]["content"] = " This is an assistant message." # extra leading space
@@ -80,7 +80,7 @@ def test_final_strip(conversation: Conversation):
"auto_leading_space": True,
"final_lstrip": False,
"final_rstrip": False,
- "default_stops": [],
+ "default_stop": [],
}
formatter = ConversationFormatter(prompt_config, DEFAULT_CHAT_TEMPLATE)
conversation.set_formatter(formatter)
diff --git a/tests/utilization/utils/test_parse_arguments.py b/tests/utilization/utils/test_parse_arguments.py
new file mode 100644
index 00000000..32c0d132
--- /dev/null
+++ b/tests/utilization/utils/test_parse_arguments.py
@@ -0,0 +1,38 @@
+import sys
+
+from ..fixtures import *
+
+sys.path.append('.')
+from utilization.utils.arguments import parse_argument
+
+
+def test_default_vllm():
+ model_args, dataset_args, evaluation_args = parse_argument(['-m', 'a-random-fake-model', '-d', 'nq', 'quac'])
+ assert model_args.model_backend == "vllm"
+ assert model_args.prefix_caching is False
+
+
+def test_no_prefix_caching():
+ # currently vllm doesn't support returning logprob for prefix caching
+ model_args, dataset_args, evaluation_args = parse_argument([
+ '-m', 'a-random-fake-model', '-d', 'nq', 'mmlu', '-b', '1'
+ ])
+ assert model_args.model_backend == "huggingface"
+ assert model_args.prefix_caching is False
+
+
+def test_default_prefix_caching():
+ # currently vllm doesn't support returning logprob for prefix caching
+ model_args, dataset_args, evaluation_args = parse_argument([
+ '-m', 'a-random-fake-model', '-d', 'nq', 'mmlu', '-b', '16'
+ ])
+ assert model_args.model_backend == "huggingface"
+ assert model_args.prefix_caching is True
+
+
+def test_default_no_efficient():
+ model_args, dataset_args, evaluation_args = parse_argument([
+ '-m', 'a-random-fake-model', '-d', 'nq', '--vllm', 'False', '--prefix_caching', 'False'
+ ])
+ assert model_args.model_backend == "huggingface"
+ assert model_args.prefix_caching is False
diff --git a/utilization/__init__.py b/utilization/__init__.py
index d3fe9a57..0ce133ac 100644
--- a/utilization/__init__.py
+++ b/utilization/__init__.py
@@ -5,6 +5,7 @@
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
# this file only initializes .utils modules to avoid early import of torch
+from .load_model import register_model
from .utils import DatasetArguments, EvaluationArguments, ModelArguments, parse_argument
if TYPE_CHECKING:
@@ -48,5 +49,6 @@ def _register_dataset_class(cls):
__all__ = [
- "get_evaluator", "parse_argument", "ModelArguments", "DatasetArguments", "EvaluationArguments", "register_dataset"
+ "get_evaluator", "parse_argument", "ModelArguments", "DatasetArguments", "EvaluationArguments", "register_dataset",
+ "register_model"
]
diff --git a/utilization/chat_templates.py b/utilization/chat_templates.py
index d0605d32..aba8727b 100644
--- a/utilization/chat_templates.py
+++ b/utilization/chat_templates.py
@@ -1,20 +1,43 @@
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List, Optional, Union
__all__ = ["DEFAULT_CHAT_TEMPLATE", "DEFAULT_CHAT_CONFIGS", "add_space", "smart_space"]
-def add_space(msg: str, auto_leading_space: bool, context: str) -> str:
- if auto_leading_space and msg and context and not context[-1].isspace() and not msg[0].isspace():
+def add_space(
+ msg: str,
+ context: str,
+ auto_leading_space: bool = True,
+ remove_space_between: bool = True,
+ starts: Optional[List[str]] = None,
+ ends: Optional[List[str]] = None
+) -> str:
+ if starts is None or ends is None or remove_space_between is False:
+ context_ends_special = False
+ msg_starts_special = False
+ else:
+ context_ends_special = any(context.endswith(e) for e in ends if len(e) > 0)
+ msg_starts_special = any(msg.startswith(s) for s in starts if len(s) > 0)
+ if (auto_leading_space and msg and context)\
+ and not (context[-1].isspace() or msg[0].isspace())\
+ and not (context_ends_special and msg_starts_special):
return ' ' + msg
return msg
-def smart_space(parts: List[str], auto_leading_space) -> str:
-
+def smart_space(parts: List[str], auto_leading_space: bool, remove_space_between: bool, seq: List[str]) -> str:
+ starts = [seq[role + "_start"] for role in ["system", "user", "assistant"]]
+ ends = [seq[role + "_end"] for role in ["system", "user", "assistant"]]
rendered = ""
for part in parts:
if part:
- rendered += add_space(part, auto_leading_space, rendered)
+ rendered += add_space(
+ part,
+ rendered,
+ auto_leading_space=auto_leading_space,
+ remove_space_between=remove_space_between,
+ starts=starts,
+ ends=ends
+ )
return rendered
@@ -37,7 +60,7 @@ def smart_space(parts: List[str], auto_leading_space) -> str:
"{%- set data.parts = data.parts + [seq['assistant_start']] -%}"
"{%- endif -%}"
""
- "{{ data.parts | smart_space(auto_leading_space) }}"
+ "{{ data.parts | smart_space(auto_leading_space, remove_space_between, seq) }}"
)
# Chat configs format:
@@ -58,7 +81,7 @@ def smart_space(parts: List[str], auto_leading_space) -> str:
# - assistant_end: The string to append to the assistant message.
# - auto_leading_space: Whether to add a leading space when concatenating two
# strings if the first string does not end with a whitespace.
-# - default_stops: A list of strings that indicate the end of a message.
+# - default_stop: A list of strings that indicate the end of a message.
#
DEFAULT_CHAT_CONFIGS: Dict[str, Union[Dict[str, Any], str]] = {
"base": {
@@ -70,7 +93,8 @@ def smart_space(parts: List[str], auto_leading_space) -> str:
"assistant_end": "\n\n",
"auto_leading_space": True,
"final_rstrip": True,
- "default_stops": [],
+ "remove_space_between": False,
+ "default_stop": [],
},
"llama2": {
"all_start": "