chore: calc n99, n90, n50 and n05 percentiles

leon0399 · Jul 31, 2024 · 66b9385 · 66b9385
1 parent 2492036
commit 66b9385
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 4 deletions.
diff --git a/README.md b/README.md
@@ -16,19 +16,19 @@ World's complete programming language benchmark.
 #### Run full suite
 
 ```bash
-docker-compose run benchmark python3 ./benchmark.py
+docker-compose run benchmark python3 ./benchmark.py run
 ```
 
 #### Run specific languages only
 
 ```bash
-docker-compose run benchmark python3 ./benchmark.py --lang rust go php
+docker-compose run benchmark python3 ./benchmark.py run --lang rust go php
 ```
 
 #### Run specific scripts only
 
 ```bash
-docker-compose run benchmark python3 ./benchmark.py --script primes/Simple linpack/Linpack recursion/Tak
+docker-compose run benchmark python3 ./benchmark.py run --script primes/Simple linpack/Linpack recursion/Tak
 ```
 
 > [!TIP]

diff --git a/benchmark.py b/benchmark.py
@@ -13,6 +13,7 @@
 import time
 import statistics
 import argparse
+import numpy
 
 parser = argparse.ArgumentParser(description='Run benchmarks')
 parser.add_argument('action', type=str, help='Action to perform', choices=['run', 'results'])
@@ -27,7 +28,7 @@
 configurations = []
 if args.languages:
     for language in args.languages:
-        configurations += glob.glob('langs' + language + '/benchmark.yml') + glob.glob('langs' + language + '/benchmark.yaml')
+        configurations += glob.glob('langs/' + language + '/benchmark.yml') + glob.glob('langs/' + language + '/benchmark.yaml')
 else:
     configurations = glob.glob('langs/*/benchmark.yml') + glob.glob('langs/*/benchmark.yaml')
 configurations.sort()
@@ -260,6 +261,11 @@ def writeResultsMarkdown(results):
                 if len(reportedResults) > 1:
                     reportedStdev = statistics.stdev(reportedResults)
 
+                reportedN99 = numpy.percentile(reportedResults, 99)
+                reportedN95 = numpy.percentile(reportedResults, 95)
+                reportedN50 = numpy.percentile(reportedResults, 50)
+                reportedN05 = numpy.percentile(reportedResults, 5)
+
                 memoryResults.sort()
                 memoryMedian = statistics.median(memoryResults)
                 memoryStdev = 0
@@ -272,6 +278,7 @@ def writeResultsMarkdown(results):
                 if len(startupResults) > 1:
                     startupStdev = statistics.stdev(startupResults)
 
+
                 result = {
                     'tags': run['tags'],
                     'total_time': {
@@ -283,6 +290,10 @@ def writeResultsMarkdown(results):
                         'results': reportedResults,
                         'median': reportedMedian,
                         'stdev': reportedStdev,
+                        'n99': reportedN99,
+                        'n95': reportedN95,
+                        'n50': reportedN50,
+                        'n05': reportedN05,
                     },
                     'startup_time': {
                         'results': startupResults,