added vllm config

configs/local_models/bert/bert.py

@@ -0,0 +1,18 @@
+from opencompass.models import Bert
+
+from mmengine.config import read_base
+with read_base():
+    from ...paths import ROOT_DIR
+
+bert_large_cased = dict(
+        type=Bert,
+        abbr='bert_large_cased',
+        path=ROOT_DIR+"models/google-bert/bert-large-cased",
+        tokenizer_path=ROOT_DIR+"models/google-bert/bert-large-cased",
+        tokenizer_kwargs=dict(trust_remote_code=True),
+        max_out_len=400,
+        max_seq_len=2048,
+        batch_size=8,
+        model_kwargs=dict(trust_remote_code=True),
+        run_cfg=dict(num_gpus=1, num_procs=1),
+    )

configs/local_models/qwen/qwen_vllm.py

@@ -0,0 +1,15 @@
+from opencompass.models import VLLM
+
+qwen_1_5b_14b_chat_vllm = dict(
+    type=VLLM,
+    abbr='qwen-1.5b-14b-chat',
+    path="/home/junetheriver/models/qwen/Qwen1.5-14B-Chat",
+    max_seq_len=2048,
+    model_kwargs=dict(trust_remote_code=True, max_model_len=2048),
+    generation_kwargs=dict(),
+    meta_template=None,
+    mode='none',
+    batch_size=1,
+    use_fastchat_template=False,
+    end_str=None,
+)

configs/tests/test_model.py

@@ -8,24 +8,22 @@
     from ..datasets.opseval.datasets import owl_mc, owl_qa
     # Models
     from ..local_models.google.t5 import t5_base
-    from ..local_models.lmsys.vicuna import vicuna_bases
-    from ..local_models.internlm.internlm import internlm2_bases
-    from ..local_models.yi.yi import yi_bases
-    from ..local_models.mistral.mistral import mistral_7b
+    from ..local_models.bert.bert import bert_large_cased
+
     from ..paths import ROOT_DIR
 
-yi_bases = [model for model in yi_bases if '34' not in model['abbr']]
 
 datasets = [
     *owl_mc, *owl_qa,
 ]
 
 datasets = [
-    dataset for dataset in datasets if 'Zero-shot' in dataset['abbr']
+    dataset for dataset in datasets if 'Zero-shot' in dataset['abbr'] and 'zh' in dataset['abbr']
 ]
 
 models = [
     t5_base,
+    # bert_large_cased,
     # *vicuna_bases,
     # *internlm2_bases,
     # *yi_bases, 

configs/tests/test_vllm.py

@@ -0,0 +1,69 @@
+from mmengine.config import read_base
+from opencompass.partitioners import SizePartitioner, NaivePartitioner
+from opencompass.runners import LocalRunner
+from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
+
+with read_base():
+    # Datasets
+    from ..datasets.opseval.datasets import owl_mc, owl_qa
+    # Models
+    from ..local_models.google.t5 import t5_base
+    from ..local_models.bert.bert import bert_large_cased
+    from ..local_models.qwen.qwen_vllm import qwen_1_5b_14b_chat_vllm
+
+    from ..paths import ROOT_DIR
+
+
+datasets = [
+    *owl_mc, *owl_qa,
+]
+
+datasets = [
+    dataset for dataset in datasets if 'Zero-shot' in dataset['abbr'] and 'zh' in dataset['abbr']
+]
+
+models = [
+    t5_base,
+    # bert_large_cased,
+    qwen_1_5b_14b_chat_vllm,
+    # *vicuna_bases,
+    # *internlm2_bases,
+    # *yi_bases, 
+    # mistral_7b
+]
+
+for model in models:
+    model['run_cfg'] = dict(num_gpus=1, num_procs=1)
+    pass
+
+for dataset in datasets:
+    dataset['sample_setting'] = dict()
+    dataset['infer_cfg']['inferencer']['save_every'] = 8
+    dataset['infer_cfg']['inferencer']['sc_size'] = 2
+    dataset['infer_cfg']['inferencer']['max_token_len'] = 20
+    dataset['eval_cfg']['sc_size'] = 2
+    dataset['sample_setting'] = dict(sample_size=2)     # !!!WARNING: Use for testing only!!!
+
+
+infer = dict(
+    partitioner=dict(
+        # type=SizePartitioner,
+        # max_task_size=100,
+        # gen_task_coef=1,
+        type=NaivePartitioner
+    ),
+    runner=dict(
+        type=LocalRunner,
+        max_num_workers=16,
+        max_workers_per_gpu=1,
+        task=dict(type=OpenICLInferTask),
+    ),
+)
+
+eval = dict(
+    partitioner=dict(type=NaivePartitioner),
+    runner=dict(
+        type=LocalRunner,
+        max_num_workers=32,
+        task=dict(type=OpenICLEvalTask)),
+)

opencompass/models/__init__.py

@@ -33,6 +33,7 @@
 # from .custom import CustomModel  # noqa
 from .wenxin_api import WenXinAI  # noqa
 from .t5 import T5  # noqa
+from .bert import Bert  # noqa
 from .pangu_api import PanGu  # noqa: F401
 from .qwen_api import Qwen  # noqa: F401
 from .sensetime_api import SenseTime  # noqa: F401