fix some import error

rdagent/benchmark/eval_method.py

@@ -12,7 +12,7 @@
     FactorTask,
     TestCase,
 )
-from rdagent.benchmark.evaluators import (
+from rdagent.factor_implementation.evolving.evaluators import (
     FactorImplementationCorrelationEvaluator,
     FactorImplementationIndexEvaluator,
     FactorImplementationIndexFormatEvaluator,

rdagent/core/task.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import Tuple
 import pandas as pd
+from rdagent.core.evolving_framework import EvolvableSubjects
 
 '''
 This file contains the all the data class for rdagent task.
@@ -66,5 +67,29 @@ def from_dict(dict):
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}[{self.factor_name}]>"
 
+class FactorImplementationList(EvolvableSubjects):
+    """
+    Factors is a list.
+    """
 
-
+    def __init__(
+        self,
+        target_factor_tasks: list[FactorTask],
+        corresponding_gt: list[TestCase] = None,
+        corresponding_gt_implementations: list[TaskImplementation] = None,
+    ):
+        super().__init__()
+        self.target_factor_tasks = target_factor_tasks
+        self.corresponding_implementations: list[TaskImplementation] = []
+        self.corresponding_selection: list[list] = []
+        self.evolve_trace = {}
+        self.corresponding_gt = corresponding_gt
+        if corresponding_gt_implementations is not None and len(
+            corresponding_gt_implementations,
+        ) != len(target_factor_tasks):
+            self.corresponding_gt_implementations = None
+            FinCoLog.warning(
+                "The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None",
+            )
+        else:
+            self.corresponding_gt_implementations = corresponding_gt_implementations

rdagent/factor_implementation/evolving/evolving_strategy.py

@@ -1,35 +1,38 @@
 from __future__ import annotations
 
 import json
-from pathlib import Path
-import random
 from abc import abstractmethod
 from copy import deepcopy
 from typing import TYPE_CHECKING
 
-from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
-from rdagent.core.utils import multiprocessing_wrapper
-from rdagent.factor_implementation.share_modules.factor import (
+from jinja2 import Template
+
+from evolving.core import EvolvingStrategy, QueriedKnowledge
+from rdagent.utils.llm import APIBackend
+from scripts.factor_implementation.share_modules.conf import FactorImplementSettings
+from scripts.factor_implementation.share_modules.factor import (
     FactorImplementation,
     FactorImplementationTask,
     FileBasedFactorImplementation,
 )
-from rdagent.core.prompts import Prompts
-from jinja2 import Template
-from rdagent.oai.llm_utils import APIBackend
 
-from rdagent.factor_implementation.share_modules.factor_implementation_config import (
-    FactorImplementSettings,
+from rdagent.factor_implementation.evolving.scheduler import (
+    RandomSelect,
+    LLMSelect,
+    LLMCoTSelect,
 )
-from rdagent.factor_implementation.share_modules.factor_implementation_utils import (
-    get_data_folder_intro,
+
+from scripts.factor_implementation.share_modules.prompt import (
+    FactorImplementationPrompts,
 )
+from scripts.factor_implementation.share_modules.utils import get_data_folder_intro
+from utils.misc import multiprocessing_wrapper
 
 if TYPE_CHECKING:
-    from factor_implementation.evolving.evolvable_subjects import (
+    from scripts.factor_implementation.baselines.evolving.evolvable_subjects import (
         FactorImplementationList,
     )
-    from factor_implementation.evolving.knowledge_management import (
+    from scripts.factor_implementation.baselines.evolving.knowledge_management import (
         FactorImplementationQueriedKnowledge,
         FactorImplementationQueriedKnowledgeV1,
     )
@@ -51,9 +54,11 @@ def evolve(
         queried_knowledge: FactorImplementationQueriedKnowledge | None = None,
         **kwargs,
     ) -> FactorImplementationList:
+        self.num_loop += 1
         new_evo = deepcopy(evo)
         new_evo.corresponding_implementations = [None for _ in new_evo.target_factor_tasks]
 
+        # 1.找出需要evolve的factor
         to_be_finished_task_index = []
         for index, target_factor_task in enumerate(new_evo.target_factor_tasks):
             target_factor_task_desc = target_factor_task.get_factor_information()
@@ -66,12 +71,34 @@ def evolve(
                 and target_factor_task_desc not in queried_knowledge.failed_task_info_set
             ):
                 to_be_finished_task_index.append(index)
+
+        # 2. 选择selection方法
+        # if the number of factors to be implemented is larger than the limit, we need to select some of them
         if FactorImplementSettings().implementation_factors_per_round < len(to_be_finished_task_index):
-            to_be_finished_task_index = random.sample(
-                to_be_finished_task_index,
-                FactorImplementSettings().implementation_factors_per_round,
-            )
+            # if the number of loops is equal to the select_loop, we need to select some of them
+            if FactorImplementSettings().select_factor_method == "random":
+                to_be_finished_task_index = RandomSelect(
+                    to_be_finished_task_index,
+                    FactorImplementSettings().implementation_factors_per_round
+                )
 
+            if FactorImplementSettings().select_factor_method == "LLM":
+                to_be_finished_task_index = LLMSelect(
+                    to_be_finished_task_index,
+                    FactorImplementSettings().implementation_factors_per_round,
+                    new_evo,
+                    queried_knowledge.all_former_traces,
+                )
+
+            if FactorImplementSettings().select_factor_method == "CoT":
+                to_be_finished_task_index = LLMCoTSelect(
+                    to_be_finished_task_index,
+                    FactorImplementSettings().implementation_factors_per_round,
+                    new_evo,
+                    queried_knowledge.all_former_traces,
+                )
+
+        # 
         result = multiprocessing_wrapper(
             [
                 (self.implement_one_factor, (new_evo.target_factor_tasks[target_index], queried_knowledge))
@@ -82,12 +109,18 @@ def evolve(
 
         for index, target_index in enumerate(to_be_finished_task_index):
             new_evo.corresponding_implementations[target_index] = result[index]
+            if result[index].target_task.factor_name in new_evo.evolve_trace:
+                new_evo.evolve_trace[result[index].target_task.factor_name].append(
+                    result[index]
+                )
+            else:
+                new_evo.evolve_trace[result[index].target_task.factor_name] = [result[index]]
 
+        new_evo.corresponding_selection.append(to_be_finished_task_index)
         # for target_index in to_be_finished_task_index:
         #     new_evo.corresponding_implementations[target_index] = self.implement_one_factor(
         #         new_evo.target_factor_tasks[target_index], queried_knowledge
         #     )
-
         return new_evo
 
 
@@ -118,9 +151,7 @@ def implement_one_factor(
             queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
 
             system_prompt = Template(
-                Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                    "evolving_strategy_factor_implementation_v1_system"
-                ],
+                FactorImplementationPrompts()["evolving_strategy_factor_implementation_v1_system"],
             ).render(
                 data_info=get_data_folder_intro(),
                 queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
@@ -133,9 +164,7 @@ def implement_one_factor(
             while True:
                 user_prompt = (
                     Template(
-                        Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                            "evolving_strategy_factor_implementation_v1_user"
-                        ],
+                        FactorImplementationPrompts()["evolving_strategy_factor_implementation_v1_user"],
                     )
                     .render(
                         factor_information_str=factor_information_str,
@@ -154,9 +183,6 @@ def implement_one_factor(
                     queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
                 elif len(queried_similar_successful_knowledge_to_render) > 1:
                     queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:]
-            # print(
-            #     f"length of queried_similar_successful_knowledge_to_render: {len(queried_similar_successful_knowledge_to_render)}, length of queried_former_failed_knowledge_to_render: {len(queried_former_failed_knowledge_to_render)}"
-            # )
 
             code = json.loads(
                 session.build_chat_completion(
@@ -172,16 +198,21 @@ def implement_one_factor(
 
             return factor_implementation
 
-
 class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
+    def __init__(self) -> None:
+        self.num_loop = 0
+        self.haveSelected = False
+
     def implement_one_factor(
         self,
         target_task: FactorImplementationTask,
         queried_knowledge,
     ) -> FactorImplementation:
         error_summary = FactorImplementSettings().v2_error_summary
+        # 1. 提取因子的背景信息
         target_factor_task_information = target_task.get_factor_information()
 
+        # 2. 检查该因子是否需要继续做（是否已经作对，是否做错太多）
         if (
             queried_knowledge is not None
             and target_factor_task_information in queried_knowledge.success_task_to_knowledge_dict
@@ -190,6 +221,8 @@ def implement_one_factor(
         elif queried_knowledge is not None and target_factor_task_information in queried_knowledge.failed_task_info_set:
             return None
         else:
+
+            # 3. 取出knowledge里面的经验数据（similar success、similar error、former_trace）
             queried_similar_component_knowledge = (
                 queried_knowledge.component_with_success_task[target_factor_task_information]
                 if queried_knowledge is not None
@@ -209,9 +242,7 @@ def implement_one_factor(
             queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
 
             system_prompt = Template(
-                Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                    "evolving_strategy_factor_implementation_v1_system"
-                ],
+                FactorImplementationPrompts()["evolving_strategy_factor_implementation_v1_system"],
             ).render(
                 data_info=get_data_folder_intro(),
                 queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
@@ -224,18 +255,17 @@ def implement_one_factor(
             queried_similar_component_knowledge_to_render = queried_similar_component_knowledge
             queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
             error_summary_critics = ""
+            # 动态地防止prompt超长
             while True:
+                # 总结error（可选）
                 if (
                     error_summary
                     and len(queried_similar_error_knowledge_to_render) != 0
                     and len(queried_former_failed_knowledge_to_render) != 0
                 ):
+
                     error_summary_system_prompt = (
-                        Template(
-                            Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                                "evolving_strategy_error_summary_v2_system"
-                            ]
-                        )
+                        Template(FactorImplementationPrompts()["evolving_strategy_error_summary_v2_system"])
                         .render(
                             factor_information_str=target_factor_task_information,
                             code_and_feedback=queried_former_failed_knowledge_to_render[
@@ -249,11 +279,7 @@ def implement_one_factor(
                     )
                     while True:
                         error_summary_user_prompt = (
-                            Template(
-                                Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                                    "evolving_strategy_error_summary_v2_user"
-                                ]
-                            )
+                            Template(FactorImplementationPrompts()["evolving_strategy_error_summary_v2_user"])
                             .render(
                                 queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
                             )
@@ -270,12 +296,10 @@ def implement_one_factor(
                         user_prompt=error_summary_user_prompt,
                         json_mode=False,
                     )
-
+                # 构建user_prompt。开始写代码
                 user_prompt = (
                     Template(
-                        Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
-                            "evolving_strategy_factor_implementation_v2_user"
-                        ],
+                        FactorImplementationPrompts()["evolving_strategy_factor_implementation_v2_user"],
                     )
                     .render(
                         factor_information_str=target_factor_task_information,
@@ -302,12 +326,6 @@ def implement_one_factor(
                 elif len(queried_similar_error_knowledge_to_render) > 0:
                     queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
 
-            # print(
-            #     len(queried_similar_component_knowledge_to_render),
-            #     len(queried_similar_error_knowledge_to_render),
-            #     len(queried_former_failed_knowledge_to_render),
-            # )
-
             response = session.build_chat_completion(
                 user_prompt=user_prompt,
                 json_mode=True,

rdagent/factor_implementation/evolving/knowledge_management.py

@@ -18,14 +18,14 @@
 )
 from rdagent.core.log import FinCoLog
 from rdagent.factor_implementation.evolving.evaluators import FactorImplementationSingleFeedback
-from rdagent.factor_implementation.share_modules.factor import (
+from rdagent.core.task import (
     FactorImplementation,
     FactorImplementationTask,
 )
 from rdagent.core.prompts import Prompts
 from rdagent.knowledge_management.graph import UndirectedGraph, UndirectedNode
 from jinja2 import Template
-from rdagent.oai.llm_utils import APIBackend, calculate_embedding_distance_between_str_list
+from rdagent.utils.llm import APIBackend, calculate_embedding_distance_between_str_list
 
 from rdagent.factor_implementation.share_modules.factor_implementation_config import (
     FactorImplementSettings,