Merge pull request #177 from macrocosm-os/dev

Release 2.7.2.
macrocosm-os · Dec 23, 2024 · 20bc7d6 · 20bc7d6
2 parents 32aba23 + c57ff6f
commit 20bc7d6
Show file tree

Hide file tree

Showing 4 changed files with 112 additions and 103 deletions.
diff --git a/constants/__init__.py b/constants/__init__.py
@@ -33,7 +33,7 @@
 # Project Constants.
 # ---------------------------------
 
-__version__ = "2.7.1"
+__version__ = "2.7.2"
 version_split = __version__.split(".")
 __spec_version__ = (
     (1000 * int(version_split[0]))
@@ -140,56 +140,12 @@
     ),
 }
 
-INSTRUCT_8B_BLOCK = 4_451_695
-IF_EVAL_V2_BLOCK = 4_523_592
+INSTRUCT_8B_TO_25_WEIGHT_BLOCK = 4_552_883
 
 # Schedule of competitions by block.
 COMPETITION_SCHEDULE_BY_BLOCK: List[Tuple[int, List[Competition]]] = [
     (
         0,
-        [
-            Competition(
-                CompetitionId.B7_MULTI_CHOICE,
-                MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.B7_MULTI_CHOICE],
-                1.0,
-                eval_tasks=[
-                    EvalTask(
-                        name="SYNTHETIC_MMLU",
-                        method_id=EvalMethodId.MULTIPLE_CHOICE,
-                        dataset_id=DatasetId.SYNTHETIC_MMLU,
-                        normalization_id=NormalizationId.NONE,
-                        weight=0.85,
-                    ),
-                    EvalTask(
-                        name="WORD_SORTING",
-                        method_id=EvalMethodId.REFERENCE_LOSS,
-                        dataset_id=DatasetId.WORD_SORTING,
-                        normalization_id=NormalizationId.INVERSE_EXPONENTIAL,
-                        normalization_kwargs={"ceiling": 40.0},
-                        weight=0.05,
-                    ),
-                    EvalTask(
-                        name="FINEWEB",
-                        method_id=EvalMethodId.TEXT_LOSS,
-                        dataset_id=DatasetId.FINEWEB,
-                        normalization_id=NormalizationId.INVERSE_EXPONENTIAL,
-                        normalization_kwargs={"ceiling": 20.0},
-                        weight=0.05,
-                    ),
-                    EvalTask(
-                        name="IF_EVAL_V1",
-                        method_id=EvalMethodId.IF_EVAL,
-                        dataset_id=DatasetId.SYNTHETIC_IF_EVAL,
-                        normalization_id=NormalizationId.NONE,
-                        dataset_kwargs={"if_eval_version": IfEvalVersion.V1},
-                        weight=0.05,
-                    ),
-                ],
-            ),
-        ],
-    ),
-    (
-        INSTRUCT_8B_BLOCK,
         [
             Competition(
                 CompetitionId.B7_MULTI_CHOICE,
@@ -201,7 +157,7 @@
                         method_id=EvalMethodId.MULTIPLE_CHOICE,
                         dataset_id=DatasetId.SYNTHETIC_MMLU,
                         normalization_id=NormalizationId.NONE,
-                        weight=0.8,
+                        weight=0.75,
                     ),
                     EvalTask(
                         name="WORD_SORTING",
@@ -220,12 +176,12 @@
                         weight=0.1,
                     ),
                     EvalTask(
-                        name="IF_EVAL_V1",
+                        name="IF_EVAL_V2",
                         method_id=EvalMethodId.IF_EVAL,
                         dataset_id=DatasetId.SYNTHETIC_IF_EVAL,
                         normalization_id=NormalizationId.NONE,
-                        dataset_kwargs={"if_eval_version": IfEvalVersion.V1},
-                        weight=0.05,
+                        dataset_kwargs={"if_eval_version": IfEvalVersion.V2},
+                        weight=0.1,
                     ),
                 ],
             ),
@@ -239,7 +195,7 @@
                         method_id=EvalMethodId.MULTIPLE_CHOICE,
                         dataset_id=DatasetId.SYNTHETIC_MMLU,
                         normalization_id=NormalizationId.NONE,
-                        weight=0.8,
+                        weight=0.75,
                     ),
                     EvalTask(
                         name="WORD_SORTING",
@@ -258,24 +214,24 @@
                         weight=0.1,
                     ),
                     EvalTask(
-                        name="IF_EVAL_V1",
+                        name="IF_EVAL_V2",
                         method_id=EvalMethodId.IF_EVAL,
                         dataset_id=DatasetId.SYNTHETIC_IF_EVAL,
                         normalization_id=NormalizationId.NONE,
-                        dataset_kwargs={"if_eval_version": IfEvalVersion.V1},
-                        weight=0.05,
+                        dataset_kwargs={"if_eval_version": IfEvalVersion.V2},
+                        weight=0.1,
                     ),
                 ],
             ),
         ],
     ),
     (
-        IF_EVAL_V2_BLOCK,
+        INSTRUCT_8B_TO_25_WEIGHT_BLOCK,
         [
             Competition(
                 CompetitionId.B7_MULTI_CHOICE,
                 MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.B7_MULTI_CHOICE],
-                0.9,
+                0.75,
                 eval_tasks=[
                     EvalTask(
                         name="SYNTHETIC_MMLU",
@@ -313,7 +269,7 @@
             Competition(
                 CompetitionId.INSTRUCT_8B,
                 MODEL_CONSTRAINTS_BY_COMPETITION_ID[CompetitionId.INSTRUCT_8B],
-                0.1,
+                0.25,
                 eval_tasks=[
                     EvalTask(
                         name="SYNTHETIC_MMLU",

diff --git a/neurons/config.py b/neurons/config.py
@@ -69,7 +69,7 @@ def validator_config():
         help="Where to store downloaded models",
     )
     parser.add_argument(
-        "--netuid", type=str, default=constants.SUBNET_UID, help="The subnet UID."
+        "--netuid", type=int, default=constants.SUBNET_UID, help="The subnet UID."
     )
     parser.add_argument(
         "--do_sample",