Merge pull request #173 from macrocosm-os/free_bump

Bump Comp 3 to 25%.
macrocosm-os · Dec 23, 2024 · e7a8c8d · e7a8c8d
2 parents cff957c + 13819ef
commit e7a8c8d
Showing 1 changed file with 12 additions and 56 deletions.
diff --git a/constants/__init__.py b/constants/__init__.py
@@ -140,56 +140,12 @@
     ),
 }
 
-INSTRUCT_8B_BLOCK = 4_451_695
-IF_EVAL_V2_BLOCK = 4_523_592
+INSTRUCT_8B_TO_25_WEIGHT_BLOCK = 4_552_883
 
 # Schedule of competitions by block.
 COMPETITION_SCHEDULE_BY_BLOCK: List[Tuple[int, List[Competition]]] = [
     (
         0,
-        [
-            Competition(
-                CompetitionId.B7_MULTI_CHOICE,
-                MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.B7_MULTI_CHOICE],
-                1.0,
-                eval_tasks=[
-                    EvalTask(
-                        name="SYNTHETIC_MMLU",
-                        method_id=EvalMethodId.MULTIPLE_CHOICE,
-                        dataset_id=DatasetId.SYNTHETIC_MMLU,
-                        normalization_id=NormalizationId.NONE,
-                        weight=0.85,
-                    ),
-                    EvalTask(
-                        name="WORD_SORTING",
-                        method_id=EvalMethodId.REFERENCE_LOSS,
-                        dataset_id=DatasetId.WORD_SORTING,
-                        normalization_id=NormalizationId.INVERSE_EXPONENTIAL,
-                        normalization_kwargs={"ceiling": 40.0},
-                        weight=0.05,
-                    ),
-                    EvalTask(
-                        name="FINEWEB",
-                        method_id=EvalMethodId.TEXT_LOSS,
-                        dataset_id=DatasetId.FINEWEB,
-                        normalization_id=NormalizationId.INVERSE_EXPONENTIAL,
-                        normalization_kwargs={"ceiling": 20.0},
-                        weight=0.05,
-                    ),
-                    EvalTask(
-                        name="IF_EVAL_V1",
-                        method_id=EvalMethodId.IF_EVAL,
-                        dataset_id=DatasetId.SYNTHETIC_IF_EVAL,
-                        normalization_id=NormalizationId.NONE,
-                        dataset_kwargs={"if_eval_version": IfEvalVersion.V1},
-                        weight=0.05,
-                    ),
-                ],
-            ),
-        ],
-    ),
-    (
-        INSTRUCT_8B_BLOCK,
         [
             Competition(
                 CompetitionId.B7_MULTI_CHOICE,
@@ -201,7 +157,7 @@
                         method_id=EvalMethodId.MULTIPLE_CHOICE,
                         dataset_id=DatasetId.SYNTHETIC_MMLU,
                         normalization_id=NormalizationId.NONE,
-                        weight=0.8,
+                        weight=0.75,
                     ),
                     EvalTask(
                         name="WORD_SORTING",
@@ -220,12 +176,12 @@
                         weight=0.1,
                     ),
                     EvalTask(
-                        name="IF_EVAL_V1",
+                        name="IF_EVAL_V2",
                         method_id=EvalMethodId.IF_EVAL,
                         dataset_id=DatasetId.SYNTHETIC_IF_EVAL,
                         normalization_id=NormalizationId.NONE,
-                        dataset_kwargs={"if_eval_version": IfEvalVersion.V1},
-                        weight=0.05,
+                        dataset_kwargs={"if_eval_version": IfEvalVersion.V2},
+                        weight=0.1,
                     ),
                 ],
             ),
@@ -239,7 +195,7 @@
                         method_id=EvalMethodId.MULTIPLE_CHOICE,
                         dataset_id=DatasetId.SYNTHETIC_MMLU,
                         normalization_id=NormalizationId.NONE,
-                        weight=0.8,
+                        weight=0.75,
                     ),
                     EvalTask(
                         name="WORD_SORTING",
@@ -258,24 +214,24 @@
                         weight=0.1,
                     ),
                     EvalTask(
-                        name="IF_EVAL_V1",
+                        name="IF_EVAL_V2",
                         method_id=EvalMethodId.IF_EVAL,
                         dataset_id=DatasetId.SYNTHETIC_IF_EVAL,
                         normalization_id=NormalizationId.NONE,
-                        dataset_kwargs={"if_eval_version": IfEvalVersion.V1},
-                        weight=0.05,
+                        dataset_kwargs={"if_eval_version": IfEvalVersion.V2},
+                        weight=0.1,
                     ),
                 ],
             ),
         ],
     ),
     (
-        IF_EVAL_V2_BLOCK,
+        INSTRUCT_8B_TO_25_WEIGHT_BLOCK,
         [
             Competition(
                 CompetitionId.B7_MULTI_CHOICE,
                 MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.B7_MULTI_CHOICE],
-                0.9,
+                0.75,
                 eval_tasks=[
                     EvalTask(
                         name="SYNTHETIC_MMLU",
@@ -313,7 +269,7 @@
             Competition(
                 CompetitionId.INSTRUCT_8B,
                 MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.INSTRUCT_8B],
-                0.1,
+                0.25,
                 eval_tasks=[
                     EvalTask(
                         name="SYNTHETIC_MMLU",