update config to v2

config/accelerate.yaml

@@ -0,0 +1,16 @@
+compute_environment: LOCAL_MACHINE
+debug: false
+distributed_type: 'NO'
+downcast_bf16: 'no'
+gpu_ids: all
+machine_rank: 0
+main_training_function: main
+mixed_precision: bf16
+num_machines: 1
+num_processes: 1
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

config/models/large.json

@@ -1,9 +1,9 @@
 {
-    "d_model": 1024,
-    "n_heads": 16,
-    "n_layers": 64,
+    "d_model": 2048,
+    "n_heads": 32,
+    "n_layers": 16,
     "ff_mult": 4,
     "drop_p": 0.0,
-    "max_seq_len": 4096,
+    "max_seq_len": 8192,
     "grad_checkpoint": true
 }

config/models/medium.json

@@ -1,9 +1,9 @@
 {
-    "d_model": 768,
-    "n_heads": 12,
-    "n_layers": 48,
+    "d_model": 1536,
+    "n_heads": 24,
+    "n_layers": 16,
     "ff_mult": 4,
     "drop_p": 0.0,
-    "max_seq_len": 4096,
+    "max_seq_len": 8192,
     "grad_checkpoint": true
-}
+}