fixes 6

turbo-llm · Nov 18, 2024 · 3d76e92 · 3d76e92
1 parent 440a515
commit 3d76e92
Show file tree

Hide file tree

Showing 12 changed files with 316 additions and 300 deletions.
diff --git a/tests/fixtures/configs/train/reinforce/deepspeed_cfg.json b/tests/fixtures/configs/train/reinforce/deepspeed_cfg.json
@@ -1,29 +1,29 @@
 {
     "fp16": {
-         "enabled": false
-     },
-     "bf16": {
+        "enabled": false
+    },
+    "bf16": {
         "enabled": true
-     },
-     "optimizer": {
-      "type": "AdamW",
-      "params": {
-          "lr": "auto",
-          "betas": "auto",
-          "eps": "auto",
-          "weight_decay": "auto"
-          }
-      },
-     "scheduler": {
-     "type": "WarmupDecayLR",
-     "params": {
-         "warmup_min_lr": "auto",
-         "warmup_max_lr": "auto",
-         "warmup_num_steps": "auto",
-         "total_num_steps": "auto"
-         }
-     },
-  
+    },
+    "optimizer": {
+    "type": "AdamW",
+    "params": {
+        "lr": "auto",
+        "betas": "auto",
+        "eps": "auto",
+        "weight_decay": "auto"
+        }
+    },
+    "scheduler": {
+    "type": "WarmupDecayLR",
+    "params": {
+        "warmup_min_lr": "auto",
+        "warmup_max_lr": "auto",
+        "warmup_num_steps": "auto",
+        "total_num_steps": "auto"
+        }
+    },
+
     "zero_optimization": {
         "stage": 3,
         "offload_param": {
@@ -34,16 +34,22 @@
             "pin_memory": true
         },
         "allgather_partitions": true,
-        "allgather_bucket_size": 2e8,
+        "allgather_bucket_size": 1e5,
         "overlap_comm": false,
         "reduce_scatter": true,
-        "reduce_bucket_size": 2e8,
-        "contiguous_gradients": true
+        "reduce_bucket_size": "auto",
+        "contiguous_gradients": true,
+        "sub_group_size": 1e5,
+        "stage3_param_persistence_threshold": "auto",
+        "stage3_prefetch_bucket_size": 0,
+        "stage3_max_live_parameters": 0,
+        "stage3_max_reuse_distance": 0,
+        "stage3_gather_16bit_weights_on_model_save": true
     },
-   "gradient_accumulation_steps": "auto",
-   "gradient_clipping": "auto",
-   "steps_per_print": 2,
-   "train_batch_size": "auto",
-   "train_micro_batch_size_per_gpu": "auto",
-   "wall_clock_breakdown": false  
+"gradient_accumulation_steps": "auto",
+"gradient_clipping": "auto",
+"steps_per_print": 2,
+"train_batch_size": "auto",
+"train_micro_batch_size_per_gpu": "auto",
+"wall_clock_breakdown": false  
 }