fixing bubugs

configs/commons/inferencers.py

@@ -1,19 +1,18 @@
 from opencompass.openicl.icl_inferencer import PPLInferencer, SCInferencer, CoTInferencer, GenInferencer
 
 
-def get_ppl_inferencer(save_every=20, fixidlist=dict(fix_id_list=None)):
+def get_ppl_inferencer(save_every=20):
     ppl_inferencer = dict(
                     type=PPLInferencer,
                     save_every=save_every,
                     infer_type='PPL',
-                    **fixidlist
+                    # **fixidlist
                 )
     return ppl_inferencer
 
 def get_gen_inferencer(save_every=20, 
                        max_out_len=400, 
                        sc_size=1, 
-                       fixidlist=dict(fix_id_list=None), 
                        generation_kwargs=dict(temperature=0.7),
                        sc=True,
                        ):
@@ -25,7 +24,6 @@ def get_gen_inferencer(save_every=20,
             infer_type='SC',
             sc_size=sc_size,  
             max_out_len=max_out_len,
-            **fixidlist
         )
     else:
         inferencer = dict(
@@ -34,11 +32,14 @@ def get_gen_inferencer(save_every=20,
             generation_kwargs=generation_kwargs,
             infer_type='Gen',
             max_out_len=max_out_len,
-            **fixidlist
         )
     return inferencer
 
-def get_cot_inferencer(save_every=20, max_out_len=400, sc_size=1, fixidlist=dict(fix_id_list=None), generation_kwargs=dict(temperature=0.7), cot_prompts=None):
+def get_cot_inferencer(save_every=20, 
+                       max_out_len=400, 
+                       sc_size=1, 
+                       generation_kwargs=dict(temperature=0.7), 
+                       cot_prompts=None):
     inferencer = dict(
                     type=CoTInferencer,
                     save_every=save_every,
@@ -47,6 +48,5 @@ def get_cot_inferencer(save_every=20, max_out_len=400, sc_size=1, fixidlist=dict
                     infer_type='SC',
                     sc_size=sc_size, 
                     max_out_len=max_out_len,
-                    **fixidlist
                 )
     return inferencer

configs/commons/templates.py

@@ -48,7 +48,7 @@ def mc_abcd_gen_prompt_template(prompt_hint, answer_hint):
                     role="HUMAN",
                     prompt=f'{prompt_hint}{{question}}\nA: {{A}}\nB: {{B}}\nC: {{C}}\nD: {{D}}\n{answer_hint}'
                 ),
-                dict(role="BOT", prompt="{answer}")
+                # dict(role="BOT", prompt="{answer}")
             ],
         ),
         ice_token="</E>",
@@ -85,7 +85,7 @@ def mc_abcd_cot_prompt_template(prompt_hint, cot_think_hint):
                     role="HUMAN",
                     prompt=f'{prompt_hint}{{question}}\nA: {{A}}\nB: {{B}}\nC: {{C}}\nD: {{D}}\n{cot_think_hint}'
                 ),
-                dict(role="BOT", prompt="{answer}")
+                # dict(role="BOT", prompt="{answer}")
             ]
         ),
         ice_token="</E>",
@@ -116,7 +116,7 @@ def qa_gen_prompt_template(prompt_hint, answer_hint):
                     role="HUMAN",
                     prompt=f'{prompt_hint}{{question}}\n{answer_hint}'
                 ),
-                dict(role="BOT", prompt="{answer}")
+                # dict(role="BOT", prompt="{answer}")
             ],
         ),
         ice_token="</E>",

configs/datasets/opseval/mc_gen.py

@@ -31,16 +31,15 @@ def get_mc_gen_datasets(dataset_name, path, langs=['zh'], qtypes=['single']):
             infer_cfg=dict(
                 ice_template=mc_abcd_gen_ice_template(prompt_hint, answer_hint),
                 prompt_template=mc_abcd_gen_prompt_template(prompt_hint, answer_hint),
-                retriever=dict(type=retriever, fix_id_list=fixidlist),
-                inferencer=get_gen_inferencer(sc_size=SAMPLE_SIZE, fixidlist=fixidlist),
+                retriever=retriever_dict,
+                inferencer=get_gen_inferencer(sc_size=SAMPLE_SIZE),
             ),
             eval_cfg=dict(evaluator=dict(type=OpsEvalGenMCEvaluator))
             )
-            for shot_abbr, fixidlist, shot_hint_id, retriever in zip(
+            for shot_abbr, shot_hint_id, retriever_dict in zip(
                 ['Zero-shot', '3-shot'],
-                [dict(fix_id_list=None), dict(fix_id_list=[0, 1, 2])],
                 [0, 1],
-                [ZeroRetriever, FixKRetriever]
+                [dict(type=ZeroRetriever), dict(type=FixKRetriever, fix_id_list=[0,1,2])]
             )
             for qtype, qtype_hint_id in zip(
                 ['single'],
@@ -68,15 +67,14 @@ def get_mc_gen_datasets(dataset_name, path, langs=['zh'], qtypes=['single']):
             infer_cfg=dict(
                 ice_template=mc_abcd_cot_ice_template(prompt_hint, cot_think_hint, cot_conclude_hint),
                 prompt_template=mc_abcd_cot_prompt_template(prompt_hint, cot_think_hint),
-                retriever=dict(type=retriever, fix_id_list=fixidlist),
-                inferencer=get_cot_inferencer(sc_size=SAMPLE_SIZE, fixidlist=fixidlist, cot_prompts=cot_conclude_hint),
+                retriever=retriever_dict,
+                inferencer=get_cot_inferencer(sc_size=SAMPLE_SIZE, cot_prompts=cot_conclude_hint),
             ),
             eval_cfg=dict(evaluator=dict(type=OpsEvalGenMCEvaluator)))
-        for shot_abbr, fixidlist, shot_hint_id, retriever in zip(
+        for shot_abbr, shot_hint_id, retriever_dict in zip(
             ['Zero-shot', '3-shot'],
-            [dict(fix_id_list=None), dict(fix_id_list=[0,1,2])],
             [0, 1],
-            [ZeroRetriever, FixKRetriever]
+            [dict(type=ZeroRetriever), dict(type=FixKRetriever, fix_id_list=[0,1,2])]
         )
         for qtype, qtype_hint_id in zip(
             ['single', 'multiple'],

configs/datasets/opseval/mc_ppl.py

@@ -32,15 +32,14 @@ def get_mc_ppl_datasets(dataset_name, path, langs=['zh'], qtypes=['single']):
             infer_cfg=dict(
                 ice_template=mc_abcd_ppl_ice_template(prompt_hint, answer_hint),
                 prompt_template=mc_abcd_ppl_prompt_template(prompt_hint, answer_hint),
-                retriever=dict(type=retriever, fix_id_list=fixidlist),
-                inferencer=get_ppl_inferencer(fixidlist=fixidlist),
+                retriever=retriever_dict,
+                inferencer=get_ppl_inferencer(),
             ),
             eval_cfg=dict(evaluator=dict(type=AccEvaluator)))
-            for shot_abbr, fixidlist, shot_hint_id, retriever in zip(
+            for shot_abbr, shot_hint_id, retriever_dict in zip(
                 ['Zero-shot', '3-shot'],
-                [dict(fix_id_list=None), dict(fix_id_list=[0, 1, 2])],
                 [0, 1],
-                [ZeroRetriever, FixKRetriever]
+                [dict(type=ZeroRetriever), dict(type=FixKRetriever, fix_id_list=[0,1,2])]
             )
             for qtype, qtype_hint_id in zip(
                 ['single'],

configs/datasets/opseval/qa_gen.py

@@ -31,16 +31,15 @@ def get_qa_gen_datasets(dataset_name, path, langs=['zh'], qtypes=None):
             infer_cfg=dict(
                 ice_template=qa_gen_ice_template(prompt_hint, answer_hint),
                 prompt_template=qa_gen_prompt_template(prompt_hint, answer_hint),
-                retriever=dict(type=retriever, fix_id_list=fixidlist),
-                inferencer=get_gen_inferencer(sc=False, fixidlist=fixidlist),
+                retriever=retriever_dict,
+                inferencer=get_gen_inferencer(sc=False),
             ),
             eval_cfg=dict(evaluator=dict(type=BleuRougeEvaluator))
             )
-            for shot_abbr, fixidlist, shot_hint_id, retriever in zip(
+            for shot_abbr, shot_hint_id, retriever_dict in zip(
                 ['Zero-shot', '3-shot'],
-                [dict(fix_id_list=None), dict(fix_id_list=[0, 1, 2])],
                 [0, 1],
-                [ZeroRetriever, FixKRetriever]
+                [dict(type=ZeroRetriever), dict(type=FixKRetriever, fix_id_list=[0,1,2])]
             )
             for lang, prompt_hint, answer_hint in zip(
                 ['zh', 'en'],

configs/datasets/opseval/qa_ppl.py

@@ -31,16 +31,15 @@ def get_qa_ppl_datasets(dataset_name, path, langs=['zh'], qtypes=None):
             infer_cfg=dict(
                 ice_template=qa_ppl_ice_template(prompt_hint, answer_hint),
                 prompt_template=qa_ppl_prompt_template(prompt_hint, answer_hint),
-                retriever=dict(type=retriever, fix_id_list=fixidlist),
-                inferencer=get_ppl_inferencer(fixidlist=fixidlist),
+                retriever=retriever_dict,
+                inferencer=get_ppl_inferencer(),
             ),
             eval_cfg=dict(evaluator=dict(type=AccEvaluator))
             )
-            for shot_abbr, fixidlist, shot_hint_id, retriever in zip(
+            for shot_abbr, shot_hint_id, retriever_dict in zip(
                 ['Zero-shot', '3-shot'],
-                [dict(fix_id_list=None), dict(fix_id_list=[0, 1, 2])],
                 [0, 1],
-                [ZeroRetriever, FixKRetriever]
+                [dict(type=ZeroRetriever), dict(type=FixKRetriever, fix_id_list=[0,1,2])]
             )
             for lang, prompt_hint, answer_hint in zip(
                 ['zh', 'en'],

configs/tests/test_merge.py

@@ -7,7 +7,7 @@
     # Datasets
     from ..datasets.opseval.datasets import owl_mc, owl_qa
     # Models
-    from ..local_models.qwen.qwen import qwen1_5_0_5b_base
+    from ..local_models.qwen.qwen import qwen1_5_4b_base
     from ..paths import ROOT_DIR
 
 datasets = [
@@ -16,16 +16,16 @@
 
 
 models = [
-    qwen1_5_0_5b_base
+    qwen1_5_4b_base
 ]
 
 for dataset in datasets:
     dataset['sample_setting'] = dict()
     dataset['infer_cfg']['inferencer']['save_every'] = 8
-    dataset['infer_cfg']['inferencer']['sc_size'] = 2
+    dataset['infer_cfg']['inferencer']['sc_size'] = 3
     dataset['infer_cfg']['inferencer']['max_token_len'] = 20
-    dataset['eval_cfg']['sc_size'] = 2
-    dataset['sample_setting'] = dict(sample_size=2)     # !!!WARNING: Use for testing only!!!
+    dataset['eval_cfg']['sc_size'] = 3
+    dataset['sample_setting'] = dict(sample_size=10)     # !!!WARNING: Use for testing only!!!
 
 
 infer = dict(

opencompass/openicl/icl_inferencer/icl_cot_inferencer.py

@@ -54,7 +54,6 @@ def __init__(
             output_json_filepath: Optional[str] = './icl_inference_output',
             output_json_filename: Optional[str] = 'predictions',
             save_every: Optional[int] = None,
-            fix_id_list: Optional[List[int]] = None,
             sc_size: Optional[int] = 1,
             infer_type: Optional[str] = '',
             cot_prompts: Optional[List[str]] = [''],
@@ -73,7 +72,6 @@ def __init__(
         self.generation_kwargs = generation_kwargs
         self.max_out_len = max_out_len
         print(f'self.max_out_len: {self.max_out_len}')
-        self.fix_id_list = fix_id_list
         self.sc_size = sc_size
         self.cot_prompts = cot_prompts
 
@@ -96,10 +94,7 @@ def inference(self,
             output_json_filename = self.output_json_filename
 
         # 2. Get results of retrieval process
-        if 'Fix' in retriever.__class__.__name__:
-            ice_idx_list = retriever.retrieve(self.fix_id_list)
-        else:
-            ice_idx_list = retriever.retrieve()
+        ice_idx_list = retriever.retrieve()
 
         # 3. Generate prompts for testing input
         prompt_list, reference_list = self.get_generation_prompt_list_from_retriever_indices(  # noqa

opencompass/openicl/icl_inferencer/icl_ppl_inferencer.py

@@ -251,7 +251,6 @@ def __init__(
             output_json_filepath: Optional[str] = './icl_inference_output',
             output_json_filename: Optional[str] = 'predictions',
             labels: Optional[List] = None,
-            fix_id_list: Optional[List[int]] = None,
             **kwargs) -> None:
         super().__init__(
             model=model,
@@ -263,7 +262,6 @@ def __init__(
         )
 
         self.labels = labels
-        self.fix_id_list = fix_id_list
 
     def inference(self,
                   retriever: BaseRetriever,
@@ -285,10 +283,7 @@ def inference(self,
             output_json_filename = self.output_json_filename
 
         # 2. Get results of retrieval process
-        if self.fix_id_list:
-            ice_idx_list = retriever.retrieve(self.fix_id_list)
-        else:
-            ice_idx_list = retriever.retrieve()
+        ice_idx_list = retriever.retrieve()
 
         # 3. Generate in-context examples for testing inputs
         #   (OpsEval) and get reference

opencompass/openicl/icl_retriever/icl_fix_k_retriever.py

@@ -43,6 +43,7 @@ def __init__(self,
     def retrieve(self):
         """Retrieve the in-context example index for each test example."""
         num_idx = len(self.index_ds)
+        print(num_idx, self.fix_id_list)
         for idx in self.fix_id_list:
             assert idx < num_idx, f'Index {idx} is out of range of {num_idx}'
         rtr_idx_list = []