From 543cd2485c142c6568145c11c8a2b32da4969f5c Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sun, 16 Jul 2023 14:15:13 -0400 Subject: [PATCH] Publish benchmarks --- .github/workflows/build.yml | 258 +++++++++++++++++- .gitignore | 3 +- examples/blocks/src/olympics/index.html | 2 + python/perspective/bench/runtime/bench.py | 10 +- .../bench/runtime/benchmark-python.html | 50 ++++ .../runtime/run_perspective_benchmark.py | 8 +- tools/perspective-bench/package.json | 2 + .../perspective-bench/src/html/benchmark.html | 2 +- tools/perspective-bench/src/js/bench.js | 12 +- tools/perspective-scripts/bench.mjs | 7 +- 10 files changed, 345 insertions(+), 9 deletions(-) create mode 100644 python/perspective/bench/runtime/benchmark-python.html diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4a313b1ce2..9a146d4dfd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -695,6 +695,120 @@ jobs: - name: WebAssembly Test run: yarn test_js --ci + ############################# ############################## + #~~~~~~~~~~~~~~~~~~~~~~~~~~~# #~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + #~~~~~~~~~~/########\~~~~~~~# #~~~~~~~~~|##########|~~~~~~~# + #~~~~~~~~~|##|~~~~|##|~~~~~~# #~~~~~~~~~|##|~~~~~~~~~~~~~~~# + #~~~~~~~~~~~~~~~~~|##|~~~~~~# #~~~~~~~~~|##|~~~~~~~~~~~~~~~# + #~~~~~~~~~~~~~|######|~~~~~~# #~~~~~~~~~|##########|~~~~~~~# + #~~~~~~~~~~~~~~~~~|##|~~~~~~# #~~~~~~~~~~~~~~~~~|##|~~~~~~~# + #~~~~~~~~~|##|~~~~|##|~~~~~~# #~~~~~~~~~|##|~~~~|##|~~~~~~~# + #~~~~~~~~~\#########/~~~~~~~# #~~~~~~~~~|##########|~~~~~~~# + #~~~~~~~~~~~~~~~~~~~~~~~~~~~# ##### #~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + # Stage Three And a Half # # . # # Benchmarking # + #~~~~~~~~~~~~~~~~~~~~~~~~~~~# ##### #~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + benchmark_js: + needs: [build_js] + if: startsWith(github.ref, 'refs/tags/v') + strategy: + matrix: + os: [ubuntu-20.04] + node-version: [18.x] + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + + ########## + # Caches # + ########## + ################ + # Yarn Cache + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT + shell: bash + if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }} + + - name: Setup yarn cache + uses: actions/cache@v3 + id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) + with: + path: | + ${{ steps.yarn-cache-dir-path.outputs.dir }} + ~/.cache/ms-playwright + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }} + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: "yarn" + cache-dependency-path: yarn.lock + + ################ + # Dependencies # + ################ + ################ + # JS + - name: Install yarn + run: npm install -g yarn + + - name: Install js dependencies + run: yarn + env: + PSP_SKIP_EMSDK_INSTALL: 1 + + #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + #~~~~~~~~~ Build Pipelines ~~~~~~~~~# + #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + - uses: actions/download-artifact@v3 + with: + name: perspective-dist + path: packages/perspective/dist/ + + - uses: actions/download-artifact@v3 + with: + name: perspective-jupyterlab-dist + path: packages/perspective-jupyterlab/dist/ + + - uses: actions/download-artifact@v3 + with: + name: perspective-viewer-d3fc-dist + path: packages/perspective-viewer-d3fc/dist + + - uses: actions/download-artifact@v3 + with: + name: perspective-viewer-datagrid-dist + path: packages/perspective-viewer-datagrid/dist + + - uses: actions/download-artifact@v3 + with: + name: perspective-viewer-openlayers-dist + path: packages/perspective-viewer-openlayers/dist + + - uses: actions/download-artifact@v3 + with: + name: perspective-workspace-dist + path: packages/perspective-workspace/dist + + - uses: actions/download-artifact@v3 + with: + name: perspective-viewer-dist + path: rust/perspective-viewer/dist + + - name: Benchmarks + run: yarn bench + + - uses: actions/upload-artifact@v3 + with: + name: perspective-js-benchmarks + path: tools/perspective-bench/dist/benchmark-js.arrow + ########################################################################################################################## ########################################################################################################################## @@ -1034,7 +1148,10 @@ jobs: if: ${{ matrix.os == 'macos-11' }} build_pyodide: - runs-on: ubuntu-22.04 + strategy: + matrix: + os: [ubuntu-20.04] + runs-on: ${{ matrix.os }} needs: [initialize] env: PYODIDE_VERSION: 0.23.2 @@ -1575,6 +1692,129 @@ jobs: ########################################################################################################################## ########################################################################################################################## + ############################ ############################## + #~~~~~~~~~~~~~~~~~~~~~~~~~~# #~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + #~~~~~~|#############|~~~~~# #~~~~~~~~~|##########|~~~~~~~# + #~~~~~~|#|~~~~~~~/##/~~~~~~# #~~~~~~~~~|##|~~~~~~~~~~~~~~~# + #~~~~~~|#|~~~~~/##/~~~~~~~~# #~~~~~~~~~|##|~~~~~~~~~~~~~~~# + #~~~~~~~~~~~~/##/~~~~~~~~~~# #~~~~~~~~~|##########|~~~~~~~# + #~~~~~~~~~~/##/~~~~~~~~~~~~# #~~~~~~~~~~~~~~~~~|##|~~~~~~~# + #~~~~~~~~/##/~~~~~~~~~~~~~~# #~~~~~~~~~|##|~~~~|##|~~~~~~~# + #~~~~~~/##/~~~~~~~~~~~~~~~~# #~~~~~~~~~|##########|~~~~~~~# + #~~~~~~~~~~~~~~~~~~~~~~~~~~# ##### #~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + # Stage Seven and a Half # # . # # Benchmarking # + #~~~~~~~~~~~~~~~~~~~~~~~~~~# ##### #~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + benchmark_python: + ###################################################################### + # Run Python build first, then use the artifacts in the python tests # + ###################################################################### + needs: [build_python] + strategy: + matrix: + os: [ubuntu-20.04] + python-version: [3.11] + node-version: [18.x] + if: startsWith(github.ref, 'refs/tags/v') + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + ########## + # Caches # + ########## + ################ + # Yarn Cache + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT + shell: bash + if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }} + + - name: Setup yarn cache + uses: actions/cache@v3 + id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) + with: + path: | + ~/.cache/ms-playwright + ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }} + + ################ + # Pip Cache + - name: Setup pip cache + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }} + + ############################### + # Language and Compiler Setup # + ############################### + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: "yarn" + cache-dependency-path: yarn.lock + + ################ + # Dependencies # + ################ + ################ + # JS + - name: Install yarn + run: npm install -g yarn + + - name: Install js dependencies + run: yarn + env: + PSP_SKIP_EMSDK_INSTALL: 1 + + ################ + # Python + - name: Install python dependencies + run: yarn _requires_python + + #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + #~~~~~~~~~ Build Pipelines ~~~~~~~~~# + #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + ################# + # Python - Test # + ################# + # Download artifact + - uses: actions/download-artifact@v3 + with: + name: perspective-python-dist-${{ matrix.os }}-${{ matrix.python-version }} + + # Install artifact in-place so tests work as-is + - name: Install wheel (Linux) + run: python -m pip install -U *manylinux2014*.whl --target python/perspective + if: ${{ runner.os == 'Linux' }} + + # Run tests + - name: Python Test Steps + run: yarn bench + + - uses: actions/upload-artifact@v3 + with: + name: perspective-python-benchmarks + path: tools/perspective-bench/dist/benchmark-python.arrow + ########################################################################################################################## + ########################################################################################################################## + ########################################## #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# #~~~~~~~~~~~~~~~/#########\~~~~~~~~~~~~~~# @@ -1596,9 +1836,14 @@ jobs: test_python_sdist, build_pyodide, test_js, + benchmark_js, + benchmark_python, ] if: startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-22.04 + strategy: + matrix: + os: [ubuntu-20.04] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 @@ -1734,6 +1979,14 @@ jobs: with: name: perspective-python-sdist + - uses: actions/download-artifact@v3 + with: + name: perspective-js-benchmarks + + - uses: actions/download-artifact@v3 + with: + name: perspective-python-benchmarks + - name: NPM pack run: | npm pack \ @@ -1760,5 +2013,6 @@ jobs: *.whl *.tar.gz *.tgz + *.arrow env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index b2595fb6ef..3ea10a78b7 100644 --- a/.gitignore +++ b/.gitignore @@ -216,4 +216,5 @@ test-results/ playwright-report/ playwright/.cache/ -.pyodide-xbuildenv \ No newline at end of file +.pyodide-xbuildenv +python/perspective/bench/runtime/benchmark_venv diff --git a/examples/blocks/src/olympics/index.html b/examples/blocks/src/olympics/index.html index b4f192d512..2350952c6c 100644 --- a/examples/blocks/src/olympics/index.html +++ b/examples/blocks/src/olympics/index.html @@ -2,8 +2,10 @@ + + diff --git a/python/perspective/bench/runtime/bench.py b/python/perspective/bench/runtime/bench.py index ce0a42d86e..a512f17eab 100644 --- a/python/perspective/bench/runtime/bench.py +++ b/python/perspective/bench/runtime/bench.py @@ -227,6 +227,14 @@ def write_results(self): with open(ARROW_PATH, "wb") as file: arrow = self._table.view().to_arrow() file.write(arrow) + + html_path = os.path.join(os.path.dirname(__file__), "benchmark-python.html") + out_path = os.path.join(os.path.dirname(ARROW_PATH), "benchmark-python.html") + + with open(html_path, "r") as input: + with open(out_path, "w") as output: + output.write(input.read()) + self._WROTE_RESULTS = True def run_method(self, func, *args, **kwargs): @@ -262,7 +270,7 @@ def run(self, version): if os.path.exists(ARROW_PATH): # if arrow exists, append to it with open(ARROW_PATH, "rb") as arr: - print("Reading table from pre-existing benchmark.arrow") + print("Reading table from pre-existing benchmark-python.arrow") self._table = Table(arr.read()) self._table.update([result]) else: diff --git a/python/perspective/bench/runtime/benchmark-python.html b/python/perspective/bench/runtime/benchmark-python.html new file mode 100644 index 0000000000..ea389e11c9 --- /dev/null +++ b/python/perspective/bench/runtime/benchmark-python.html @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/python/perspective/bench/runtime/run_perspective_benchmark.py b/python/perspective/bench/runtime/run_perspective_benchmark.py index 10efb4d5e6..4d1d525d62 100644 --- a/python/perspective/bench/runtime/run_perspective_benchmark.py +++ b/python/perspective/bench/runtime/run_perspective_benchmark.py @@ -17,7 +17,13 @@ if __name__ == "__main__": """Benchmark the `perspective-python` runtime locally.""" - VERSIONS = ["master", "2.2.1", "2.2.0"] + VERSIONS = [ + "master", + "2.3.1", + # "2.3.0", + "2.2.1", + "2.2.0", + ] # Access the benchmark virtualenv HERE = os.path.abspath(os.path.dirname(__file__)) diff --git a/tools/perspective-bench/package.json b/tools/perspective-bench/package.json index 89a6bfd5bb..7389512754 100644 --- a/tools/perspective-bench/package.json +++ b/tools/perspective-bench/package.json @@ -22,6 +22,8 @@ "microtime": "^3.0.0" }, "dependencies": { + "perspective-2-3-0": "npm:@finos/perspective@2.3.0", + "perspective-2-2-0": "npm:@finos/perspective@2.2.0", "perspective-2-1-0": "npm:@finos/perspective@2.1.0", "perspective-2-0-0": "npm:@finos/perspective@2.0.0", "perspective-1-9-0": "npm:@finos/perspective@1.9.0", diff --git a/tools/perspective-bench/src/html/benchmark.html b/tools/perspective-bench/src/html/benchmark.html index d20c9b6bf2..2501b3b2fa 100644 --- a/tools/perspective-bench/src/html/benchmark.html +++ b/tools/perspective-bench/src/html/benchmark.html @@ -37,7 +37,7 @@ window.addEventListener("DOMContentLoaded", async () => { const el = document.getElementsByTagName("perspective-viewer")[0]; el.addEventListener("perspective-config-update", async () => localStorage.setItem("layout", JSON.stringify(await el.save()))); - const req = await fetch("benchmark.arrow"); + const req = await fetch("benchmark-js.arrow"); const arrow = await req.arrayBuffer(); const worker = perspective.worker(); const table = await worker.table(arrow); diff --git a/tools/perspective-bench/src/js/bench.js b/tools/perspective-bench/src/js/bench.js index 67babc3f62..f89c4ba179 100644 --- a/tools/perspective-bench/src/js/bench.js +++ b/tools/perspective-bench/src/js/bench.js @@ -34,11 +34,21 @@ async function persist_to_arrow(obs_records) { const view = await table.view(); await table.update(obs_records); const arrow = await view.to_arrow(); + if (!fs.existsSync(path.join(__dirname, "../../dist/"))) { + fs.mkdirSync(path.join(__dirname, "../../dist/")); + } + fs.writeFileSync( - path.join(__dirname, "../../dist/benchmark.arrow"), + path.join(__dirname, "../../dist/benchmark-js.arrow"), Buffer.from(arrow), "binary" ); + + fs.writeFileSync( + path.join(__dirname, "../../dist/benchmark.html"), + fs.readFileSync(path.join(__dirname, "../html/benchmark.html")), + "binary" + ); } /** diff --git a/tools/perspective-scripts/bench.mjs b/tools/perspective-scripts/bench.mjs index fa1ac9d708..ed3bb57a5b 100644 --- a/tools/perspective-scripts/bench.mjs +++ b/tools/perspective-scripts/bench.mjs @@ -17,9 +17,12 @@ dotenv.config({ path: "./.perspectiverc" }); const args = process.argv.slice(2); if (process.env.PSP_PROJECT === undefined || process.env.PSP_PROJECT === "js") { - sh`yarn && sudo nice -n -20 npm run bench` + sh`yarn && nice -n 0 npm run bench` .cwd("tools/perspective-bench") .runSync(); } else { - sh`python3 ${args}`.env({ PYTHONPATH: "./python/perspective" }).runSync(); + sh`PYTHONPATH=python/perspective nice -n 0 python3 python/perspective/bench/runtime/run_perspective_benchmark.py` + // .env({ PYTHONPATH: "python/perspective" }) + .log() + .runSync(); }