Merge pull request #8 from tbvanderwoude/moving-ai-benchmarks

Standard Moving AI pathfinding benchmarks

.gitignore

@@ -1,2 +1,4 @@
 target/
 .idea
+scenarios/
+maps/

Cargo.toml

@@ -25,6 +25,7 @@ bench = false
 
 [dev-dependencies]
 criterion = { version = "0.4", features = ["html_reports"] }
+grid_pathfinding_benchmark = { path = "grid_pathfinding_benchmark" }
 rand = "0.8.5"
 
 [[bench]]

README.md

@@ -3,10 +3,10 @@
 A grid-based pathfinding system. Implements [Jump Point Search](https://en.wikipedia.org/wiki/Jump_point_search) with 
 [improved pruning rules](https://www.researchgate.net/publication/287338108_Improving_jump_point_search) for speedy pathfinding. Pre-computes
 [connected components](https://en.wikipedia.org/wiki/Component_(graph_theory))
-to avoid flood-filling behaviour if no path exists. Both [4-neighborhood](https://en.wikipedia.org/wiki/Von_Neumann_neighborhood) and [8-neighborhood](https://en.wikipedia.org/wiki/Moore_neighborhood) grids are supported and a custom variant of JPS is implemented for the 4-neighborhood. 
+to avoid flood-filling behavior if no path exists. Both [4-neighborhood](https://en.wikipedia.org/wiki/Von_Neumann_neighborhood) and [8-neighborhood](https://en.wikipedia.org/wiki/Moore_neighborhood) grids are supported and a custom variant of JPS is implemented for the 4-neighborhood. 
 
 ### Example
-Below a [simple 8-grid example](examples/simple_8.rs) is given which illustrates how to set a basic problem and find a path.
+Below a [simple 8-grid example](examples/simple_8.rs) is given, illustrating how to set a basic problem and find a path.
 ```rust,no_run
 use grid_pathfinding::PathingGrid;
 use grid_util::grid::Grid;
@@ -36,21 +36,23 @@ fn main() {
     println!("Path:");
     for p in path {
         println!("{:?}", p);
-    }
+ }
 }
 ```
 This assumes an 8-neighborhood, which is the default grid type. The same problem can be solved for a 4-neighborhood, disallowing diagonal moves, by adding the line
 ```rust,no_run
 pathing_grid.allow_diagonal_move = false;
 ```
-prior to component generation, which is done in example [simple_4](examples/simple_4.rs).
+before component generation, which is done in example [simple_4](examples/simple_4.rs).
 
 
 
 See [examples](examples/) for finding paths with multiple goals and generating waypoints instead of full paths.
 
 ### Benchmarks
-To run a set of benchmarks, use `cargo bench`. A baseline can be set using
+The system can be benchmarked using scenarios from the [Moving AI 2D pathfinding benchmarks](https://movingai.com/benchmarks/grids.html). The [grid_pathfinding_benchmark](grid_pathfinding_benchmark) utility crate provides general support for loading these files. The default benchmark executed using `cargo bench` runs three scenario sets from the [Dragon Age: Origins](https://movingai.com/benchmarks/dao/index.html): `dao/arena`, `dao/den312` and `dao/arena2` (or `dao/den009d` when using the rectilinear algorithm). Running these requires the corresponding map and scenario files to be saved in folders called `maps/dao` and `scenarios/dao`.
+
+A baseline can be set using
 ```bash
 cargo bench -- --save-baseline main
 ```
@@ -59,6 +61,10 @@ New runs can be compared to this baseline using
 cargo bench -- --baseline main
 ```
 
+### Performance
+Using an i5-6600 quad-core running at 3.3 GHz, running the `dao/arena2` set takes 134 ms using JPS allowing diagonals and with improved pruning disabled. Using default neighbor generation as in normal A* (enabled by setting `GRAPH_PRUNING = false`) makes this take 1.37 s, a factor 10 difference. As a rule, the relative difference increases as maps get larger, with the `dao/arena` set (a smaller map) taking 846 us and 1.34 ms respectively with and without pruning. 
+
+An existing C++ [JPS implementation](https://github.com/nathansttt/hog2) runs the same scenarios in about 60 ms. The fastest known solver is the [l1-path-finder](https://mikolalysenko.github.io/l1-path-finder/www/) (implemented in Javascript) which can do this in only 38 ms using A* with landmarks (for a 4-neighborhood). This indicates that there is still a lot of room for improvement in terms of search speed.
 
 ### Goal of crate
 The long-term goal of this crate is to provide a fast off-the-shelf pathfinding implementation for grids.

benches/comparison_bench.rs

@@ -1,83 +1,37 @@
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use grid_pathfinding::PathingGrid;
+use grid_pathfinding_benchmark::*;
 use grid_util::grid::Grid;
-use grid_util::point::Point;
-use rand::prelude::*;
 
-fn random_grid(
-    n: usize,
-    rng: &mut StdRng,
-    allow_diag: bool,
-    pruning: bool,
-    fill_rate: f64,
-) -> PathingGrid {
-    let mut pathing_grid: PathingGrid = PathingGrid::new(n, n, false);
-    pathing_grid.allow_diagonal_move = allow_diag;
-    pathing_grid.improved_pruning = pruning;
-    for x in 0..pathing_grid.width() {
-        for y in 0..pathing_grid.height() {
-            pathing_grid.set(x, y, rng.gen_bool(fill_rate))
-        }
-    }
-    pathing_grid.generate_components();
-    pathing_grid
-}
-fn random_grid_point(grid: &PathingGrid, rng: &mut StdRng) -> Point {
-    Point::new(
-        rng.gen_range(0..grid.width()) as i32,
-        rng.gen_range(0..grid.height()) as i32,
-    )
-}
-
-fn test(pathing_grid: &PathingGrid, start: Point, end: Point) -> Option<Vec<Point>> {
-    black_box(pathing_grid.get_path_single_goal(start, end, false))
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
+fn dao_bench(c: &mut Criterion) {
     for (allow_diag, pruning) in [(false, false), (true, false), (true, true)] {
-        const N: usize = 64;
-        const N_GRIDS: usize = 1000;
-        const N_PAIRS: usize = 1000;
-        let mut rng = StdRng::seed_from_u64(0);
-        let mut random_grids: Vec<PathingGrid> = Vec::new();
-        for _ in 0..N_GRIDS {
-            random_grids.push(random_grid(N, &mut rng, allow_diag, pruning, 0.4))
-        }
+        let bench_set = if allow_diag {
+            ["dao/arena", "dao/den312d", "dao/arena2"]
+        } else {
+            ["dao/arena", "dao/den009d", "dao/den312d"]
+        };
+        for name in bench_set {
+            let (bool_grid, scenarios) = get_benchmark(name.to_owned());
+            let mut pathing_grid: PathingGrid =
+                PathingGrid::new(bool_grid.width, bool_grid.height, true);
+            pathing_grid.grid = bool_grid.clone();
+            pathing_grid.allow_diagonal_move = allow_diag;
+            pathing_grid.improved_pruning = pruning;
+            pathing_grid.update_all_neighbours();
+            pathing_grid.generate_components();
+            let diag_str = if allow_diag { "8-grid" } else { "4-grid" };
+            let improved_str = if pruning { " (improved pruning)" } else { "" };
 
-        let start = Point::new(0, 0);
-        let end = Point::new(N as i32 - 1, N as i32 - 1);
-        let diag_str = if allow_diag { "8-grid" } else { "4-grid" };
-        let improved_str = if pruning { " (improved pruning)" } else { "" };
-        c.bench_function(
-            format!("1000 random 64x64 {diag_str}s{improved_str}").as_str(),
-            |b| {
+            c.bench_function(format!("{name}, {diag_str}{improved_str}").as_str(), |b| {
                 b.iter(|| {
-                    for grid in &random_grids {
-                        test(grid, start, end);
+                    for (start, end) in &scenarios {
+                        black_box(pathing_grid.get_path_single_goal(*start, *end, false));
                     }
                 })
-            },
-        );
-        let grid = &random_grids[0];
-        let mut random_pairs: Vec<(Point, Point)> = Vec::new();
-        for _ in 0..N_PAIRS {
-            random_pairs.push((
-                random_grid_point(&grid, &mut rng),
-                random_grid_point(&grid, &mut rng),
-            ))
+            });
         }
-        c.bench_function(
-            format!("1000 random start goal pairs on a 64x64 {diag_str}{improved_str}").as_str(),
-            |b| {
-                b.iter(|| {
-                    for (start, end) in &random_pairs {
-                        test(&grid, start.clone(), end.clone());
-                    }
-                })
-            },
-        );
     }
 }
 
-criterion_group!(benches, criterion_benchmark);
+criterion_group!(benches, dao_bench);
 criterion_main!(benches);

benches/single_bench.rs

@@ -0,0 +1,33 @@
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use grid_pathfinding::PathingGrid;
+use grid_pathfinding_benchmark::*;
+use grid_util::grid::Grid;
+
+fn dao_bench_single(c: &mut Criterion) {
+    for (allow_diag, pruning) in [(true, false)] {
+        let bench_set = ["dao/arena"];
+        for name in bench_set {
+            let (bool_grid, scenarios) = get_benchmark(name.to_owned());
+            let mut pathing_grid: PathingGrid =
+                PathingGrid::new(bool_grid.width, bool_grid.height, true);
+            pathing_grid.grid = bool_grid.clone();
+            pathing_grid.allow_diagonal_move = allow_diag;
+            pathing_grid.improved_pruning = pruning;
+            pathing_grid.update_all_neighbours();
+            pathing_grid.generate_components();
+            let diag_str = if allow_diag { "8-grid" } else { "4-grid" };
+            let improved_str = if pruning { " (improved pruning)" } else { "" };
+
+            c.bench_function(format!("{name}, {diag_str}{improved_str}").as_str(), |b| {
+                b.iter(|| {
+                    for (start, end) in &scenarios {
+                        black_box(pathing_grid.get_path_single_goal(*start, *end, false));
+                    }
+                })
+            });
+        }
+    }
+}
+
+criterion_group!(benches, dao_bench_single);
+criterion_main!(benches);

examples/benchmark_runner.rs

@@ -0,0 +1,43 @@
+use grid_pathfinding::PathingGrid;
+use grid_pathfinding_benchmark::*;
+use grid_util::grid::Grid;
+use grid_util::point::Point;
+use std::time::{Duration, Instant};
+
+fn main() {
+    let benchmark_names = get_benchmark_names();
+    let mut total_time = Duration::ZERO;
+    for name in benchmark_names {
+        println!("Benchmark name: {}", name);
+
+        let (bool_grid, scenarios) = get_benchmark(name);
+        // for (allow_diag, pruning) in [(false, false), (true, false), (true, true)] {
+        for (allow_diag, pruning) in [(true, false)] {
+            let mut pathing_grid: PathingGrid =
+                PathingGrid::new(bool_grid.width, bool_grid.height, true);
+            pathing_grid.grid = bool_grid.clone();
+            pathing_grid.allow_diagonal_move = allow_diag;
+            pathing_grid.improved_pruning = pruning;
+            pathing_grid.update_all_neighbours();
+            pathing_grid.generate_components();
+            let number_of_scenarios = scenarios.len() as u32;
+            let before = Instant::now();
+            run_scenarios(&pathing_grid, &scenarios);
+            let elapsed = before.elapsed();
+            println!(
+                "\tElapsed time: {:.2?}; per scenario: {:.2?}",
+                elapsed,
+                elapsed / number_of_scenarios
+            );
+            total_time += elapsed;
+        }
+    }
+    println!("\tTotal benchmark time: {:.2?}", total_time);
+}
+
+pub fn run_scenarios(pathing_grid: &PathingGrid, scenarios: &Vec<(Point, Point)>) {
+    for (start, goal) in scenarios {
+        let path: Option<Vec<Point>> = pathing_grid.get_waypoints_single_goal(*start, *goal, false);
+        assert!(path.is_some());
+    }
+}

grid_pathfinding_benchmark/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "grid_pathfinding_benchmark"
+version = "0.1.0"
+authors = ["Thom van der Woude <[email protected]>"]
+edition = "2021"
+description = "Helper crate for loading Moving AI pathfinding benchmarks"
+keywords = ["pathfinding","grid","benchmark"]
+categories = ["game-development","simulation","algorithms"]
+license = "MIT"
+repository = "https://github.com/tbvanderwoude/grid_pathfinding"
+readme = "README.md"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+grid_util = "0.1.1"
+criterion = { version = "0.4", features = ["html_reports"] }
+csv = "1.3.0"
+serde = "1.0.204"
+walkdir = "2.5.0"

grid_pathfinding_benchmark/README.md

@@ -0,0 +1,2 @@
+# grid_pathfinding_benchmark
+Helper crate for loading Moving AI pathfinding benchmarks.