feat(granite): Add model params for granite-code-8b

Something isn't quite working with this model yet, but the config should be accurate at this point. Branch: GraniteCodeSupport Signed-off-by: Gabe Goodhart <[email protected]>
pytorch · Oct 31, 2024 · b0bc180 · b0bc180
1 parent e24b7e3
commit b0bc180
Showing 1 changed file with 16 additions and 0 deletions.
diff --git a/torchchat/model_params/Granite-8B-Code.json b/torchchat/model_params/Granite-8B-Code.json
@@ -0,0 +1,16 @@
+{
+    "block_size": 128000,
+    "dim": 4096,
+    "hidden_dim": 14336,
+    "n_heads": 32,
+    "n_local_heads": 8,
+    "n_layers": 36,
+    "rope_base": 10000000,
+    "vocab_size": 49152,
+    "use_hf_tokenizer": true,
+    "norm_eps": 0.00001,
+    "rope_scaling": null,
+    "attention_bias": true,
+    "feed_forward_bias": true,
+    "tie_word_embeddings": true
+}