added ragas to OpsEvalGenQAEvaluator

.gitignore

@@ -11,6 +11,7 @@ configs/openai_key.py
 configs/paths.py
 configs/api_key.py
 configs/secrets.py
+configs/ragas_config.toml
 configs/datasets/log.json
 configs/eval_debug*.py
 configs/viz_*.py

configs/datasets/opseval/qa_gen.py

@@ -2,7 +2,7 @@
 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import FixKRetriever, ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer, SCInferencer, CoTInferencer
-from opencompass.openicl.icl_evaluator import AccEvaluator, BleuRougeEvaluator, OpsEvalGenQAEvaluator
+from opencompass.openicl.icl_evaluator import AccEvaluator, BleuRougeEvaluator, OpsEvalGenQAEvaluator, OpsEvalRagasEvaluator
 from opencompass.utils.text_postprocessors import first_capital_postprocess_multi
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import PPLInferencer

configs/lyh/claude.py

@@ -18,7 +18,9 @@
 ]
 
 # Remove COT and 3-shot datasets
-datasets = [dataset for dataset in datasets if 'cot' not in dataset['abbr'] and '3-shot' not in dataset['abbr']]
+# datasets = [dataset for dataset in datasets if 'cot' not in dataset['abbr'] and '3-shot' not in dataset['abbr']]
+# Naive COT only
+datasets = [dataset for dataset in datasets if 'cot' in dataset['abbr'] and '3-shot' not in dataset['abbr']]
 
 models = [
     claude_3_opus

configs/lyh/vllm_qa_1.py

@@ -62,6 +62,6 @@
     partitioner=dict(type=NaivePartitioner),
     runner=dict(
         type=LocalRunner,
-        max_num_workers=32,
+        max_num_workers=1,
         task=dict(type=OpenICLEvalTask)),
 )

opencompass/openicl/icl_evaluator/opseval_gen_evaluator.py

@@ -126,16 +126,19 @@ def __init__(self, language='en'):
         super().__init__()
         self.language = language
 
-    def score(self, predictions: List, references: List) -> dict:
+    def score(self, predictions: List, references: List, test_set: List) -> dict:
         tot_bleu, tot_rouge = 0, 0
         for pred, ans in zip(predictions, references):
             bleu_score, rouge_score = self.get_rouge_bleu(pred, ans, self.language)
             tot_bleu += bleu_score
             tot_rouge += rouge_score
-        return {
+        ragas_report = self.get_ragas_score(predictions, references, test_set)
+        report = {
             "bleu": tot_bleu / len(predictions),
-            "rouge": tot_rouge / len(predictions)
+            "rouge": tot_rouge / len(predictions),
+            "ragas": ragas_report 
         }
+        return report
 
     def get_rouge_bleu(self, pred, answer, language='en'):
         rouge = Rouge()
@@ -157,16 +160,25 @@ def clean_word(words):
             bleu_score, rouge_score = 0.0, 0.0
         return bleu_score, rouge_score
 
+    def get_ragas_score(self, predictions, references, test_set) -> dict:
+        from opencompass.ragas.judge import calculate_score
+        reference = [{"id": idx, "question": question, "answer": ref} 
+                            for idx, (question, ref) in enumerate(zip(test_set['question'], references))]
+        answers = [{"id": idx, "question": question, "answer": ans} 
+                            for idx, (question, ans) in enumerate(zip(test_set['question'], predictions))]
+        report = calculate_score(reference, answers)
+        return report
+
 class OpsEvalRagasEvaluator(BaseEvaluator):
 
-    def __init__(self):
+    def __init__(self, language='en'):
         super().__init__()
 
-    def score(self, predictions: List, references: List) -> dict:
-        from opencompass.ragas.judge import calculate_scores
-        reference = {"id": idx, "answer": ref
-                            for idx, ref in enumerate(references)}
-        answers = {"id": idx, "answer": ans
-                            for idx, ans in enumerate(predictions)}
-        report = calculate_scores(reference, answers)
+    def score(self, predictions: List, references: List, test_set: List) -> dict:
+        from opencompass.ragas.judge import calculate_score
+        reference = [{"id": idx, "question": question, "answer": ref} 
+                            for idx, (question, ref) in enumerate(zip(test_set['question'], references))]
+        answers = [{"id": idx, "question": question, "answer": ans} 
+                            for idx, (question, ans) in enumerate(zip(test_set['question'], predictions))]
+        report = calculate_score(reference, answers)
         return report

opencompass/ragas/__init__.py

@@ -1,3 +1,3 @@
-from config import *
-from judge import *
-from metric import *
+from .config import *
+from .judge import *
+from .metric import *

opencompass/ragas/config.py

@@ -4,15 +4,17 @@
 from pip._vendor import tomli
 from langchain_core.language_models import BaseLanguageModel
 from langchain_core.embeddings import Embeddings
+from mmengine.config import read_base
 
-CONFIG_PATH = os.getenv('CONFIG_PATH', 'config/config.toml')
+CURRENT_PATH = os.path.dirname(__file__)
+CONFIG_PATH = os.path.join(CURRENT_PATH, '../../configs', 'ragas_config.toml')
 
 logger = logging.getLogger(__name__)
 
 
 def load_config() -> dict:
     if not os.path.exists(CONFIG_PATH):
-        logger.error('Config file does not exist')
+        logger.error(f'Config file {CONFIG_PATH} does not exist')
         sys.exit(1)
 
     with open(CONFIG_PATH, 'r') as f:

opencompass/ragas/judge.py

@@ -3,7 +3,7 @@
 import logging
 import argparse
 import pandas as pd
-from config import config, load_llm, load_embeddings
+from opencompass.ragas.config import config, load_llm, load_embeddings
 from datasets import Dataset
 
 logger = logging.getLogger(__name__)
@@ -141,7 +141,7 @@ def compute_scores(df: pd.DataFrame) -> list[dict]:
         os.environ["LANGCHAIN_ENDPOINT"] = langsmith_config.get('endpoint')
 
     from ragas import evaluate, RunConfig
-    from metric import answer_correctness
+    from opencompass.ragas.metric import answer_correctness
     from langchain.callbacks.tracers import LangChainTracer
 
     new_df = df.copy()