diff --git a/.gitattributes b/.gitattributes index 0a17c695..17442a16 100644 --- a/.gitattributes +++ b/.gitattributes @@ -17,6 +17,7 @@ # Files to exclude from asset-lib download. /addons/gd-plug export-ignore /default_env.tres export-ignore +/benchmark export-ignore /docs export-ignore /.env.example export-ignore /examples export-ignore diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d6cbd6d7..88376b86 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -312,6 +312,28 @@ jobs: name: failed-screenshots path: test/visual_regression/screenshots + benchmark: + name: Benchmark + runs-on: ubuntu-latest + needs: [build-linux] + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install just + uses: taiki-e/install-action@just + - name: Import assets + shell: bash + run: godot --editor --headless --quit-after 100 || true + - name: Install binary build artifacts + uses: actions/download-artifact@v4 + with: + path: addons/godot_xterm/native/bin + merge-multiple: true + - name: Benchmark + shell: bash + run: just benchmark + merge-artifacts: name: Merge Artifacts runs-on: ubuntu-latest diff --git a/.gitmodules b/.gitmodules index 5a977fa1..0c376d68 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "addons/godot_xterm/native/thirdparty/node-pty"] path = addons/godot_xterm/native/thirdparty/node-pty url = https://github.com/microsoft/node-pty +[submodule "benchmark/vtebench"] + path = benchmark/vtebench + url = git@github.com:alacritty/vtebench diff --git a/Justfile b/Justfile index 7114a3b0..a53daa11 100644 --- a/Justfile +++ b/Justfile @@ -32,3 +32,6 @@ test-visual: uninstall: {{godot}} --headless -s plug.gd uninstall + +benchmark: + ./benchmark/editor_launch.sh {{godot}} && ls -1 benchmark/vtebench/benchmarks/ | xargs -I {} {{godot}} --windowed --resolution 800x600 --position 0,0 benchmark/benchmark.tscn -- --benchmark={} diff --git a/benchmark/benchmark.gd b/benchmark/benchmark.gd new file mode 100644 index 00000000..855db07e --- /dev/null +++ b/benchmark/benchmark.gd @@ -0,0 +1,121 @@ +extends Control + + +class Results: + var render_cpu := 0.0 + var render_gpu := 0.0 + var vtebench := {value = 0.0, variance = 0.0} + + +var terminal_exit_code := -1 + + +func _ready(): + var timeout := 120 + var benchmark := "" + + var args = OS.get_cmdline_user_args() + for arg in args: + if arg.begins_with("--benchmark"): + benchmark = arg.split("=")[1] + + if benchmark.is_empty(): + _quit_with_error("No benchmark specified") + + RenderingServer.viewport_set_measure_render_time(get_tree().root.get_viewport_rid(), true) + + var results := Results.new() + var begin_time := Time.get_ticks_usec() + var frames_captured := 0 + + $Terminal.run_benchmark(benchmark) + await $Terminal.started + + while terminal_exit_code == -1: + await get_tree().process_frame + + if Time.get_ticks_usec() - begin_time > (timeout * 1e6): + _quit_with_error("Benchmark took longer than %ss to run" % timeout) + + results.render_cpu += ( + RenderingServer.viewport_get_measured_render_time_cpu( + get_tree().root.get_viewport_rid() + ) + + RenderingServer.get_frame_setup_time_cpu() + ) + results.render_gpu += RenderingServer.viewport_get_measured_render_time_gpu( + get_tree().root.get_viewport_rid() + ) + + if terminal_exit_code != 0: + _quit_with_error("Terminal exited with error code: %d" % terminal_exit_code) + + results.render_cpu /= float(max(1.0, float(frames_captured))) + results.render_gpu /= float(max(1.0, float(frames_captured))) + + results.vtebench = _process_dat_results("res://benchmark/results/%s.dat" % benchmark) + + var json_results = ( + JSON + . stringify( + [ + { + name = benchmark, + value = _round(results.vtebench.value), + range = results.vtebench.range, + }, + { + name = "%s - render cpu" % benchmark, + value = _round(results.render_cpu), + }, + { + name = "%s - render gpu" % benchmark, + value = _round(results.render_gpu), + } + ], + " " + ) + ) + + var file = FileAccess.open("res://benchmark/results/%s.json" % benchmark, FileAccess.WRITE) + file.store_string(json_results) + + print(json_results) + get_tree().quit(terminal_exit_code) + + +func _on_terminal_exited(exit_code: int): + terminal_exit_code = exit_code + + +func _round(val: float, sig_figs := 4) -> float: + return snapped(val, pow(10, floor(log(val) / log(10)) - sig_figs + 1)) + + +func _process_dat_results(path: String) -> Dictionary: + var file := FileAccess.open(path, FileAccess.READ) + var samples := [] + + file.get_line() # Skip the first 'header' line. + while !file.eof_reached(): + var line := file.get_line().strip_edges() + if line.is_valid_float(): + samples.append(line.to_float()) + + if samples.size() < 2: + _quit_with_error("Not enough samples") + + var avg: float = (samples.reduce(func(acc, n): return acc + n, 0)) / samples.size() + + var std_dev := 0.0 + for sample in samples: + std_dev += pow(sample - avg, 2) + std_dev /= (samples.size() - 1) + + return {value = avg, range = "± %.2f" % _round(sqrt(std_dev))} + + +func _quit_with_error(error_msg: String): + await get_tree().process_frame + push_error(error_msg) + get_tree().quit(1) diff --git a/benchmark/benchmark.tscn b/benchmark/benchmark.tscn new file mode 100644 index 00000000..6978b67d --- /dev/null +++ b/benchmark/benchmark.tscn @@ -0,0 +1,26 @@ +[gd_scene load_steps=5 format=3 uid="uid://b2axn64mqnt8n"] + +[ext_resource type="Script" path="res://benchmark/benchmark.gd" id="1_tmqb5"] +[ext_resource type="PackedScene" uid="uid://cysad55lwtnc6" path="res://examples/terminal/terminal.tscn" id="2_3raq0"] +[ext_resource type="Script" path="res://benchmark/terminal_benchmark.gd" id="3_8t8od"] +[ext_resource type="FontVariation" uid="uid://ckq73bs2fwsie" path="res://themes/fonts/regular.tres" id="3_hnrrm"] + +[node name="Benchmark" type="Control"] +layout_mode = 3 +anchors_preset = 15 +anchor_right = 1.0 +anchor_bottom = 1.0 +grow_horizontal = 2 +grow_vertical = 2 +script = ExtResource("1_tmqb5") + +[node name="Terminal" parent="." instance=ExtResource("2_3raq0")] +layout_mode = 1 +focus_mode = 0 +theme_override_fonts/normal_font = ExtResource("3_hnrrm") +script = ExtResource("3_8t8od") + +[connection signal="exited" from="Terminal" to="." method="_on_terminal_exited"] +[connection signal="data_received" from="Terminal/PTY" to="Terminal" method="_on_pty_data_received"] + +[editable path="Terminal"] diff --git a/benchmark/editor_launch.sh b/benchmark/editor_launch.sh new file mode 100755 index 00000000..867189d8 --- /dev/null +++ b/benchmark/editor_launch.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -e + +godot=${1:-godot} + +if ! command -v $godot &> /dev/null; then + echo "Error: '$godot' command not found. Please provide a valid path to the Godot executable." + exit 1 +fi + +results_file=benchmark/results/editor_launch.json +value=$({ time -p $godot --editor --quit; } 2>&1 | tail -n3 | head -n1 | cut -d' ' -f2) +cat < $results_file +[ + { + "name": "editor_launch", + "value": $value + } +] +EOF +cat $results_file + diff --git a/benchmark/results/.gitignore b/benchmark/results/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/benchmark/results/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/benchmark/terminal_benchmark.gd b/benchmark/terminal_benchmark.gd new file mode 100644 index 00000000..bcb42457 --- /dev/null +++ b/benchmark/terminal_benchmark.gd @@ -0,0 +1,31 @@ +extends "res://examples/terminal/terminal.gd" + +signal started +signal exited(exit_code: int) + +var vtebench_dir := ProjectSettings.globalize_path("res://benchmark/vtebench") + + +func _ready(): + pty.connect("exited", self._on_exit) + + +func run_benchmark(benchmark): + pty.fork( + "cargo", + ["run", "--", "-b", "benchmarks/%s" % benchmark, "--dat", "../results/%s.dat" % benchmark], + vtebench_dir, + 87, + 29 + ) + + +func _on_exit(exit_code, _signal): + exited.emit(exit_code) + + +func _on_pty_data_received(data: PackedByteArray): + # Listen for the reset sequence (\x1bc), to determine that the benchmark has started. + if data.slice(0, 2) == PackedByteArray([27, 99]): + $PTY.disconnect("data_received", _on_pty_data_received) + started.emit() diff --git a/benchmark/vtebench b/benchmark/vtebench new file mode 160000 index 00000000..c75155bf --- /dev/null +++ b/benchmark/vtebench @@ -0,0 +1 @@ +Subproject commit c75155bfc252227c0efc101c1971df3e327c71c4