add 7b model

PrimeIntellect-ai · Jul 23, 2024 · 1a72a9a · 1a72a9a
1 parent 3c9c0e4
commit 1a72a9a
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/open_diloco/configs/config_7b.json b/open_diloco/configs/config_7b.json
@@ -0,0 +1,10 @@
+{
+    "name": "llama7b",
+    "n_embd": 4096,
+    "intermediate_size": 11008,
+    "n_head": 32,
+    "n_layer": 32,
+    "n_query_groups": null,
+    "vocab_size": 32000,
+    "block_size": 1024
+}