diff --git a/results.json b/results.json index 9df2bc9..8a0b56a 100644 --- a/results.json +++ b/results.json @@ -6,7 +6,7 @@ "model_type":"CLM", "num_params":13000000000.0, "num_tokens":2000000000000.0, - "size_pretraining_GB":"N/A", + "size_pretraining_GB":"-", "source":"https://huggingface.co/meta-llama/Llama-2-13b-hf", "accuracy":{ "mean":0.6691674212, @@ -20,7 +20,7 @@ "model_type":"CLM", "num_params":7000000000.0, "num_tokens":2000000000000.0, - "size_pretraining_GB":"N/A", + "size_pretraining_GB":"-", "source":"https://huggingface.co/meta-llama/Llama-2-7b-hf", "accuracy":{ "mean":0.6239813737, @@ -33,8 +33,8 @@ "model_family":"mistral", "model_type":"CLM", "num_params":7000000000.0, - "num_tokens":"N/A", - "size_pretraining_GB":"N/A", + "num_tokens":"-", + "size_pretraining_GB":"-", "source":"https://huggingface.co/mistralai/Mistral-7B-v0.1/discussions/8", "accuracy":{ "mean":0.6537317294, @@ -48,7 +48,7 @@ "model_type":"MLM", "num_params":109000000.0, "num_tokens":3300000000.0, - "size_pretraining_GB":"N/A", + "size_pretraining_GB":"-", "source":"https://huggingface.co/blog/bert-101", "accuracy":{ "mean":0.1839348079, @@ -62,7 +62,7 @@ "model_type":"MLM", "num_params":335000000.0, "num_tokens":3300000000.0, - "size_pretraining_GB":"N/A", + "size_pretraining_GB":"-", "source":"https://huggingface.co/blog/bert-101", "accuracy":{ "mean":0.199111801, @@ -76,7 +76,7 @@ "model_type":"CLM", "num_params":2000000000.0, "num_tokens":2000000000000.0, - "size_pretraining_GB":"N/A", + "size_pretraining_GB":"-", "source":"https://cloud.google.com/blog/products/ai-machine-learning/performance-deepdive-of-gemma-on-google-cloud", "accuracy":{ "mean":0.5153710171, @@ -90,7 +90,7 @@ "model_type":"CLM", "num_params":7000000000.0, "num_tokens":6000000000000.0, - "size_pretraining_GB":"N/A", + "size_pretraining_GB":"-", "source":"https://huggingface.co/docs/transformers/main/en/model_doc/gemma", "accuracy":{ "mean":0.6372181262, @@ -103,7 +103,7 @@ "model_family":"gpt2", "model_type":"CLM", "num_params":137000000.0, - "num_tokens":"N/A", + "num_tokens":"-", "size_pretraining_GB":"40.0", "source":"https://openai.com/index/better-language-models/", "accuracy":{ @@ -117,7 +117,7 @@ "model_family":"gpt2", "model_type":"CLM", "num_params":812000000.0, - "num_tokens":"N/A", + "num_tokens":"-", "size_pretraining_GB":"40.0", "source":"https://openai.com/index/better-language-models/", "accuracy":{ @@ -131,7 +131,7 @@ "model_family":"gpt2", "model_type":"CLM", "num_params":355000000.0, - "num_tokens":"N/A", + "num_tokens":"-", "size_pretraining_GB":"40.0", "source":"https://openai.com/index/better-language-models/", "accuracy":{ @@ -145,7 +145,7 @@ "model_family":"gpt2", "model_type":"CLM", "num_params":1600000000.0, - "num_tokens":"N/A", + "num_tokens":"-", "size_pretraining_GB":"40.0", "source":"https://openai.com/index/better-language-models/", "accuracy":{ @@ -257,7 +257,7 @@ "model_family":"roberta", "model_type":"MLM", "num_params":125000000.0, - "num_tokens":"N/A", + "num_tokens":"-", "size_pretraining_GB":"160.0", "source":"https://arxiv.org/pdf/1907.11692", "accuracy":{ @@ -271,7 +271,7 @@ "model_family":"roberta", "model_type":"MLM", "num_params":355000000.0, - "num_tokens":"N/A", + "num_tokens":"-", "size_pretraining_GB":"160.0", "source":"https://arxiv.org/pdf/1907.11692", "accuracy":{ @@ -285,7 +285,7 @@ "model_family":"xlm-roberta", "model_type":"MLM", "num_params":279000000.0, - "num_tokens":"N/A", + "num_tokens":"-", "size_pretraining_GB":"2500.0", "source":"https://huggingface.co/FacebookAI/xlm-roberta-base", "accuracy":{ @@ -299,7 +299,7 @@ "model_family":"xlm-roberta", "model_type":"MLM", "num_params":561000000.0, - "num_tokens":"N/A", + "num_tokens":"-", "size_pretraining_GB":"2500.0", "source":"https://huggingface.co/FacebookAI/xlm-roberta-large", "accuracy":{ @@ -313,8 +313,8 @@ "model_family":"baseline", "model_type":"-", "num_params":"-", - "num_tokens":"N/A", - "size_pretraining_GB":"N/A", + "num_tokens":"-", + "size_pretraining_GB":"-", "source":"", "accuracy":{ "mean":0.0468244729 @@ -327,7 +327,7 @@ "model_type": "CLM", "num_params": 8000000000, "num_tokens":15000000000000.0, - "size_pretraining_GB":"N/A", + "size_pretraining_GB":"-", "source":"https://huggingface.co/meta-llama/Meta-Llama-3-8B", "accuracy": { "mean": 0.68594, @@ -340,8 +340,8 @@ "model_family": "llama-3", "model_type": "CLM-IT", "num_params": 8000000000, - "num_tokens":"N/A", - "size_pretraining_GB":"N/A", + "num_tokens":"-", + "size_pretraining_GB":"-", "source":"https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct", "accuracy": { "mean": 0.64084,