Skip to content

Add CI requirements update script #1518

Add CI requirements update script

Add CI requirements update script #1518

Workflow file for this run

name: Build Status
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~ Build Triggers ~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
on:
push:
branches:
- master
tags:
- v*
paths-ignore:
- AUTHORS
- CHANGELOG.md
- CONTRIBUTING.md
- LICENSE
- README.md
- binder/
- docs/
- examples/
- python/perspective/README.md
pull_request:
branches:
- master
workflow_dispatch:
inputs:
ci-full:
description: "Run Full CI"
required: false
type: boolean
default: false
ci-skip-cache:
description: "Omit Cache from CI run"
required: false
type: boolean
default: false
ci-skip-python:
description: "Skip Python components of CI"
required: false
type: boolean
default: false
ci-include-windows:
description: "Include Windows (Python) components of CI"
required: false
type: boolean
default: false
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~ Build Matrix ~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
jobs:
##########################################################################################################################
##########################################################################################################################
####################################
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~/##########\~~~~~~~~~~~~#
#~~~~~~~~~|#|~~~~~~~|#|~~~~~~~~~~~~#
#~~~~~~~~~|#|~~~~~~~|#|~~~~~~~~~~~~#
#~~~~~~~~~|#|~~~~~~~|#|~~~~~~~~~~~~#
#~~~~~~~~~|#|~~~~~~~|#|~~~~~~~~~~~~#
#~~~~~~~~~|#|~~~~~~~|#|~~~~~~~~~~~~#
#~~~~~~~~~\##########/~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# Stage One - Initialize the build #
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# This is so we can inspect the latest commit message from
# both push and pull_request events (there is no
# github.event.head_commit.message otherwise on pull
# requests)
initialize:
runs-on: ubuntu-20.04
# Map a step output to a job output
outputs:
COMMIT_MESSAGE: ${{ steps.setup.outputs.COMMIT_MSG }}
SKIP_CI: ${{ steps.setuppush.outputs.SKIP_CI || steps.setuppr.outputs.SKIP_CI || steps.setupmanual.outputs.SKIP_CI }}
SKIP_CACHE: ${{ steps.setuppush.outputs.SKIP_CACHE || steps.setuppr.outputs.SKIP_CACHE || steps.setupmanual.outputs.SKIP_CACHE }}
FULL_RUN: ${{ steps.setuppush.outputs.FULL_RUN || steps.setuppr.outputs.FULL_RUN || steps.setupmanual.outputs.FULL_RUN }}
SKIP_PYTHON: ${{ steps.setuppush.outputs.SKIP_PYTHON || steps.setuppr.outputs.SKIP_PYTHON || steps.setupmanual.outputs.SKIP_PYTHON }}
INCLUDE_WINDOWS: ${{ steps.setuppush.outputs.INCLUDE_WINDOWS || steps.setuppr.outputs.INCLUDE_WINDOWS || steps.setupmanual.outputs.INCLUDE_WINDOWS }}
steps:
- name: Checkout
uses: actions/checkout@v3
with:
# for pull_request so we can do HEAD^2
fetch-depth: 2
- name: Get Commit Message
run: echo "COMMIT_MSG=$(git log -1 --pretty=%B HEAD | tr '\n' ' ')" >> $GITHUB_ENV
if: ${{ github.event_name == 'push' }}
- name: Get Commit Message
run: echo "COMMIT_MSG=$(git log -1 --pretty=%B HEAD^2 | tr '\n' ' ')" >> $GITHUB_ENV
if: ${{ github.event_name == 'pull_request' }}
- name: Display and Setup Build Args (Push)
id: setuppush
run: |
echo "Commit Message: $COMMIT_MSG"
echo "Skip CI: $SKIP_CI"
echo "Skip Cache: $SKIP_CACHE"
echo "Full Run: $FULL_RUN"
echo "Skip Python: $SKIP_PYTHON"
echo "Include Windows: $INCLUDE_WINDOWS"
echo "COMMIT_MSG=$COMMIT_MSG" >> $GITHUB_OUTPUT
echo "SKIP_CI=$SKIP_CI" >> $GITHUB_OUTPUT
echo "SKIP_CACHE=$SKIP_CACHE" >> $GITHUB_OUTPUT
echo "FULL_RUN=$FULL_RUN" >> $GITHUB_OUTPUT
echo "SKIP_PYTHON=$SKIP_PYTHON" >> $GITHUB_OUTPUT
echo "INCLUDE_WINDOWS=$INCLUDE_WINDOWS" >> $GITHUB_OUTPUT
env:
SKIP_CI: ${{ contains(github.event.head_commit.message, '[ci-skip]') }}
SKIP_CACHE: ${{ contains(github.event.head_commit.message, '[ci-skip-cache]') }}
FULL_RUN: ${{ startsWith(github.ref_name, 'v') || contains(github.event.head_commit.message, '[ci-full]') }}
SKIP_PYTHON: ${{ contains(github.event.head_commit.message, '[ci-skip-python]') }}
INCLUDE_WINDOWS: ${{ contains(github.event.head_commit.message, '[ci-include-windows]') }}
if: ${{ github.event_name == 'push' }}
- name: Display and Setup Build Args (PR)
id: setuppr
run: |
echo "Commit Message: $COMMIT_MSG"
echo "Skip CI: $SKIP_CI"
echo "Skip Cache: $SKIP_CACHE"
echo "Full Run: $FULL_RUN"
echo "Skip Python: $SKIP_PYTHON"
echo "Include Windows: $INCLUDE_WINDOWS"
echo "COMMIT_MSG=$COMMIT_MSG" >> $GITHUB_OUTPUT
echo "SKIP_CI=$SKIP_CI" >> $GITHUB_OUTPUT
echo "SKIP_CACHE=$SKIP_CACHE" >> $GITHUB_OUTPUT
echo "FULL_RUN=$FULL_RUN" >> $GITHUB_OUTPUT
echo "SKIP_PYTHON=$SKIP_PYTHON" >> $GITHUB_OUTPUT
echo "INCLUDE_WINDOWS=$INCLUDE_WINDOWS" >> $GITHUB_OUTPUT
env:
SKIP_CI: ${{ contains(github.event.pull_request.title, '[ci-skip]') || contains(env.COMMIT_MSG, '[ci-skip]') }}
SKIP_CACHE: ${{ contains(github.event.pull_request.title, '[ci-skip-cache]') || contains(env.COMMIT_MSG, '[ci-skip-cache]') }}
FULL_RUN: ${{ contains(github.event.pull_request.title, '[ci-full]') || contains(env.COMMIT_MSG, '[ci-full]') }}
SKIP_PYTHON: ${{ contains(github.event.pull_request.title, '[ci-skip-python]') || contains(env.COMMIT_MSG, '[ci-skip-python]') }}
INCLUDE_WINDOWS: ${{ contains(github.event.pull_request.title, '[ci-include-windows]') || contains(env.COMMIT_MSG, '[ci-include-windows]') }}
if: ${{ github.event_name == 'pull_request' }}
- name: Display and Setup Build Args (Manual)
id: setupmanual
run: |
echo "Commit Message: $COMMIT_MSG"
echo "Skip CI: $SKIP_CI"
echo "Skip Cache: $SKIP_CACHE"
echo "Full Run: $FULL_RUN"
echo "Skip Python: $SKIP_PYTHON"
echo "Include Windows: $INCLUDE_WINDOWS"
echo "COMMIT_MSG=$COMMIT_MSG" >> $GITHUB_OUTPUT
echo "SKIP_CI=$SKIP_CI" >> $GITHUB_OUTPUT
echo "SKIP_CACHE=$SKIP_CACHE" >> $GITHUB_OUTPUT
echo "FULL_RUN=$FULL_RUN" >> $GITHUB_OUTPUT
echo "SKIP_PYTHON=$SKIP_PYTHON" >> $GITHUB_OUTPUT
echo "INCLUDE_WINDOWS=$INCLUDE_WINDOWS" >> $GITHUB_OUTPUT
env:
SKIP_CI: false
SKIP_CACHE: ${{ github.event.inputs.ci-skip-cache }}
FULL_RUN: ${{ github.event.inputs.ci-full }}
SKIP_PYTHON: ${{ github.event.inputs.ci-skip-python }}
INCLUDE_WINDOWS: ${{ github.event.inputs.ci-include-windows }}
if: ${{ github.event_name == 'workflow_dispatch' }}
##########################################################################################################################
##########################################################################################################################
###################################
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~/#######|~~~~~~~~~~~~~#
#~~~~~~~~~~/########|~~~~~~~~~~~~~#
#~~~~~~~~~/###/~|###|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~|###|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~|###|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~|###|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~|###|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# Stage One - Build the JS Assets #
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
build_js:
needs: [initialize]
strategy:
matrix:
os:
- ubuntu-20.04 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md
python-version:
- 3.9
node-version: [18.x]
# NOTE: don't fail fast as sometimes npm blocks the burst of fetches from GHA
fail-fast: false
##################################
# Setup matrix job configuration #
##################################
runs-on: ${{ matrix.os }}
if: ${{ needs.initialize.outputs.SKIP_CI == 'false' }}
steps:
- name: Checkout
uses: actions/checkout@v3
##########
# Caches #
##########
################
# Yarn Cache
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
- name: Setup yarn cache
uses: actions/cache@v3
id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
with:
path: |
~/.cache/ms-playwright
${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
- name: Setup emsdk cache
uses: actions/cache@v3
id: emsdk-cache
with:
path: .emsdk
key: ${{ runner.os }}-emsdk-${{ hashFiles('package.json') }}
restore-keys: |
${{ runner.os }}-emsdk-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
################
# Pip Cache
- name: Setup pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
################
# Cargo Cache
- name: Setup cargo cache
uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
rust/perspective-viewer/target/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
################
# Build Assets
- name: ccache
uses: hendrikmuhs/[email protected]
with:
key: ${{ github.job }}-${{ matrix.os }}
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
###############################
# Language and Compiler Setup #
###############################
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
cache: "yarn"
cache-dependency-path: yarn.lock
- name: Install latest nightly rust
uses: dtolnay/rust-toolchain@nightly
with:
toolchain: nightly-2023-04-07
targets: wasm32-unknown-unknown
components: rustfmt, clippy, rust-src
################
# Dependencies #
################
################
# JS
- name: Install yarn
run: npm install -g yarn
- name: Install js dependencies
run: yarn
################
# Python
- name: Install python dependencies
run: yarn _requires_python
################
# Linux
- name: Linux init steps
run: sudo node tools/perspective-scripts/install_tools.mjs
if: ${{ runner.os == 'Linux' }}
- name: Install CCache
run: sudo apt install -y ccache
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~ Build Pipelines ~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
###############
# WebAssembly #
###############
- name: WebAssembly Lint
run: yarn lint
- name: WebAssembly Build
run: yarn build_js --ci
env:
PSP_USE_CCACHE: 1
#################################
# Compress and upload Artifacts #
#################################
######################################
# Upload built JS artifacts for tests
#
# (listed here in the same order they appear in the directory)
- uses: actions/upload-artifact@v3
with:
name: perspective-dist
path: packages/perspective/dist/
- uses: actions/upload-artifact@v3
with:
name: perspective-jupyterlab-dist
path: packages/perspective-jupyterlab/dist/
- uses: actions/upload-artifact@v3
with:
name: perspective-viewer-d3fc-dist
path: packages/perspective-viewer-d3fc/dist
- uses: actions/upload-artifact@v3
with:
name: perspective-viewer-datagrid-dist
path: packages/perspective-viewer-datagrid/dist
- uses: actions/upload-artifact@v3
with:
name: perspective-viewer-openlayers-dist
path: packages/perspective-viewer-openlayers/dist
- uses: actions/upload-artifact@v3
with:
name: perspective-esbuild-plugin-dist
path: packages/perspective-esbuild-plugin/dist
- uses: actions/upload-artifact@v3
with:
name: perspective-webpack-plugin-dist
path: packages/perspective-webpack-plugin/dist
- uses: actions/upload-artifact@v3
with:
name: perspective-cli-dist
path: packages/perspective-cli/dist
- uses: actions/upload-artifact@v3
with:
name: perspective-workspace-dist
path: packages/perspective-workspace/dist
- uses: actions/upload-artifact@v3
with:
name: perspective-viewer-dist
path: rust/perspective-viewer/dist
######################################################
# Upload Jupyter artifacts for python build/test/dist
- uses: actions/upload-artifact@v3
with:
name: nbextension-dist
path: python/perspective/perspective/nbextension
if-no-files-found: error # TODO
- uses: actions/upload-artifact@v3
with:
name: labextension-dist
path: python/perspective/perspective/labextension
if-no-files-found: error # TODO
##########################################################################################################################
##########################################################################################################################
#######################################
#~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~/########\~~~~~~~#
#~~~~~~|###|~~~\###\~~~~~~#
#~~~~~~~~~~~~~/###/~~~~~~~#
#~~~~~~~~~~~/###/~~~~~~~~~#
#~~~~~~~~~/###/~~~~~~~~~~~#
#~~~~~~~/###/~~~~~~~~~~~~~#
#~~~~~|#############|~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~#
# Stage Two - Lint Python #
#~~~~~~~~~~~~~~~~~~~~~~~~~#
lint_python:
needs: [initialize]
strategy:
matrix:
os:
- ubuntu-20.04 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md
python-version:
- 3.9
# NOTE: don't fail fast as sometimes npm blocks the burst of fetches from GHA
fail-fast: false
##################################
# Setup matrix job configuration #
##################################
runs-on: ${{ matrix.os }}
if: ${{ needs.initialize.outputs.SKIP_CI == 'false' && needs.initialize.outputs.SKIP_PYTHON == 'false' }}
steps:
- name: Checkout
uses: actions/checkout@v3
##########
# Caches #
##########
################
# Yarn Cache
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
- name: Setup yarn cache
uses: actions/cache@v3
id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
with:
path: |
~/.cache/ms-playwright
${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
################
# Pip Cache
- name: Setup pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
###############################
# Language and Compiler Setup #
###############################
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
cache: "yarn"
cache-dependency-path: yarn.lock
################
# Dependencies #
################
################
# JS
- name: Install yarn
run: npm install -g yarn
- name: Install js dependencies
run: yarn
env:
PSP_SKIP_EMSDK_INSTALL: 1
################
# Python
- name: Install python dependencies
run: yarn _requires_python
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~ Build Pipelines ~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#################
# Python - Lint #
#################
########
# Linux
- name: Python Lint Steps (Linux)
run: yarn lint
env:
PYTHON_VERSION: ${{ matrix.python-version }}
##########################################################################################################################
##########################################################################################################################
#############################
#~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~/########\~~~~~~~#
#~~~~~~~~~|##|~~~~|##|~~~~~~#
#~~~~~~~~~~~~~~~~~|##|~~~~~~#
#~~~~~~~~~~~~~|######|~~~~~~#
#~~~~~~~~~~~~~~~~~|##|~~~~~~#
#~~~~~~~~~|##|~~~~|##|~~~~~~#
#~~~~~~~~~\#########/~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# Stage Three - Test the JS #
#~~~~~~~~~~~~~~~~~~~~~~~~~~~#
test_js:
needs: [initialize, build_js]
strategy:
matrix:
os:
- ubuntu-20.04 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md
python-version:
- 3.9
node-version: [18.x]
# NOTE: don't fail fast as sometimes npm blocks the burst of fetches from GHA
fail-fast: false
##################################
# Setup matrix job configuration #
##################################
runs-on: ${{ matrix.os }}
if: ${{ needs.initialize.outputs.SKIP_CI == 'false' }}
steps:
- name: Checkout
uses: actions/checkout@v3
##########
# Caches #
##########
################
# Yarn Cache
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
- name: Setup yarn cache
uses: actions/cache@v3
id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
with:
path: |
${{ steps.yarn-cache-dir-path.outputs.dir }}
~/.cache/ms-playwright
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
################
# Pip Cache
- name: Setup pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
################
# Cargo Cache
- name: Setup cargo cache
uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
rust/perspective-viewer/target/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
###############################
# Language and Compiler Setup #
###############################
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
cache: "yarn"
cache-dependency-path: yarn.lock
################
# Dependencies #
################
################
# JS
- name: Install yarn
run: npm install -g yarn
- name: Install js dependencies
run: yarn
env:
PSP_SKIP_EMSDK_INSTALL: 1
################
# Python
- name: Install python dependencies
run: yarn _requires_python
################
# Rust
- name: Install latest nightly rust
uses: dtolnay/rust-toolchain@nightly
with:
toolchain: nightly-2023-04-07
targets: wasm32-unknown-unknown
components: rustfmt, clippy, rust-src
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~ Build Pipelines ~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
- uses: actions/download-artifact@v3
with:
name: perspective-dist
path: packages/perspective/dist/
- uses: actions/download-artifact@v3
with:
name: perspective-jupyterlab-dist
path: packages/perspective-jupyterlab/dist/
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-d3fc-dist
path: packages/perspective-viewer-d3fc/dist
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-datagrid-dist
path: packages/perspective-viewer-datagrid/dist
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-openlayers-dist
path: packages/perspective-viewer-openlayers/dist
- uses: actions/download-artifact@v3
with:
name: perspective-workspace-dist
path: packages/perspective-workspace/dist
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-dist
path: rust/perspective-viewer/dist
###############
# WebAssembly #
###############
- name: WebAssembly Test
run: yarn test_js --ci
############################# ##############################
#~~~~~~~~~~~~~~~~~~~~~~~~~~~# #~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~/########\~~~~~~~# #~~~~~~~~~|##########|~~~~~~~#
#~~~~~~~~~|##|~~~~|##|~~~~~~# #~~~~~~~~~|##|~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~|##|~~~~~~# #~~~~~~~~~|##|~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~|######|~~~~~~# #~~~~~~~~~|##########|~~~~~~~#
#~~~~~~~~~~~~~~~~~|##|~~~~~~# #~~~~~~~~~~~~~~~~~|##|~~~~~~~#
#~~~~~~~~~|##|~~~~|##|~~~~~~# #~~~~~~~~~|##|~~~~|##|~~~~~~~#
#~~~~~~~~~\#########/~~~~~~~# #~~~~~~~~~|##########|~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~# ##### #~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# Stage Three And a Half # # . # # Benchmarking #
#~~~~~~~~~~~~~~~~~~~~~~~~~~~# ##### #~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
benchmark_js:
needs: [build_js]
if: startsWith(github.ref, 'refs/tags/v')
strategy:
matrix:
os: [ubuntu-20.04]
node-version: [18.x]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@v3
##########
# Caches #
##########
################
# Yarn Cache
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
- name: Setup yarn cache
uses: actions/cache@v3
id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
with:
path: |
${{ steps.yarn-cache-dir-path.outputs.dir }}
~/.cache/ms-playwright
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
cache: "yarn"
cache-dependency-path: yarn.lock
################
# Dependencies #
################
################
# JS
- name: Install yarn
run: npm install -g yarn
- name: Install js dependencies
run: yarn
env:
PSP_SKIP_EMSDK_INSTALL: 1
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~ Build Pipelines ~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
- uses: actions/download-artifact@v3
with:
name: perspective-dist
path: packages/perspective/dist/
- uses: actions/download-artifact@v3
with:
name: perspective-jupyterlab-dist
path: packages/perspective-jupyterlab/dist/
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-d3fc-dist
path: packages/perspective-viewer-d3fc/dist
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-datagrid-dist
path: packages/perspective-viewer-datagrid/dist
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-openlayers-dist
path: packages/perspective-viewer-openlayers/dist
- uses: actions/download-artifact@v3
with:
name: perspective-workspace-dist
path: packages/perspective-workspace/dist
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-dist
path: rust/perspective-viewer/dist
- name: Benchmarks
run: yarn bench
- uses: actions/upload-artifact@v3
with:
name: perspective-js-benchmarks
path: tools/perspective-bench/dist/benchmark-js.arrow
##########################################################################################################################
##########################################################################################################################
########################################
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~|##|~~~~|##|~~~~~~~~~~~~~~#
#~~~~~~~~~~~~|##|~~~~|##|~~~~~~~~~~~~~~#
#~~~~~~~~~~~~|##|~~~~|##|~~~~~~~~~~~~~~#
#~~~~~~~~~~~~|##########|~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~|##|~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~|##|~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~|##|~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# Stage Four - Build the Python Assets #
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
build_python:
###################################################################
# Run JS build first, then use the artifacts in the python builds #
###################################################################
needs: [initialize, build_js]
strategy:
# NOTE: this matrix should build wheels across all supported envs.
# For each entry, a corresponding test is included below that
# will download the built wheel and run the test suite.
#
# If you update anything here, including the `excludes`, ensure
# that the subsequent test matrix takes those changes into account.
matrix:
os:
- ubuntu-20.04 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md
- macos-11 # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11-Readme.md
- windows-2022 # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2022-Readme.md
python-version:
- 3.7
- 3.8
- 3.9
- "3.10"
- 3.11
container:
- none # Not manylinux, we will use this space to build the WASM assets
- 2014
node-version: [18.x]
is-full-run:
- ${{ needs.initialize.outputs.FULL_RUN == 'true' }}
include-windows-run:
- ${{ needs.initialize.outputs.INCLUDE_WINDOWS == 'true' || needs.initialize.outputs.FULL_RUN == 'true' }}
exclude:
############################
# Things to always exclude #
############################
# Exclude manylinux python builds other than linux
- os: macos-11
container: 2014
- os: windows-2022
container: 2014
# Exclude non-manylinux builds on linux
- os: ubuntu-20.04
container: none
##############################################
# Things to exclude if not a full matrix run #
##############################################
# Exclude windows builds if set
- include-windows-run: false
os: windows-2022
- is-full-run: false
os: macos-11
# Exclude Python 3.7, 3.8, 3.10, 3.11 builds
- is-full-run: false
python-version: 3.7
- is-full-run: false
python-version: 3.8
- is-full-run: false
python-version: "3.10"
- is-full-run: false
python-version: 3.11
# NOTE: don't fail fast as sometimes npm blocks the burst of fetches from GHA
fail-fast: false
##################################
# Setup matrix job configuration #
##################################
runs-on: ${{ matrix.os }}
if: ${{ needs.initialize.outputs.SKIP_CI == 'false' && needs.initialize.outputs.SKIP_PYTHON == 'false' }}
steps:
- name: Checkout
uses: actions/checkout@v3
#####################################################
# Conditionals #
# --------------------------------------------------#
# Here is a reference for conditionals used #
# #
# Operating system: #
# if: ${{ runner.os == 'Linux' }} #
# if: ${{ runner.os == 'macOS' }} #
# if: ${{ runner.os == 'Windows' }} #
# #
#####################################################
################
# JS Artifacts #
################
- uses: actions/download-artifact@v3
with:
name: nbextension-dist
path: python/perspective/perspective/nbextension
- uses: actions/download-artifact@v3
with:
name: labextension-dist
path: python/perspective/perspective/labextension
##########
# Caches #
##########
################
# Yarn Cache
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
- name: Setup yarn cache
uses: actions/cache@v3
id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
with:
path: |
${{ steps.yarn-cache-dir-path.outputs.dir }}
~/.cache/ms-playwright
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
################
# Pip Cache
- name: Setup pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
if: ${{ runner.os != 'Linux' && needs.initialize.outputs.SKIP_CACHE == 'false' }}
################
# Homebrew Cache
- name: Setup homebrew cache
uses: actions/cache@v3
with:
path: |
~/Library/Caches/Homebrew/boost--*
~/Library/Caches/Homebrew/downloads/*--boost-*
key: brew-${{ hashFiles('cpp/perspective/CMakeLists.txt') }}
restore-keys: brew-
if: ${{ runner.os == 'macOS' && needs.initialize.outputs.SKIP_CACHE == 'false' }}
################
# vcpkg Cache
- name: Setup vcpkg cache
uses: actions/cache@v3
with:
path: |
C:\Users\runneradmin\AppData\Local\vcpkg\archives
C:\vcpkg\packages\
C:\vcpkg\downloads\
C:\vcpkg\buildtrees\
C:\vcpkg\ports\
key: vcpkg-${{ hashFiles('cpp/perspective/CMakeLists.txt') }}
restore-keys: vcpkg-
if: ${{ runner.os == 'Windows' && needs.initialize.outputs.SKIP_CACHE == 'false' }}
###############################
# Language and Compiler Setup #
###############################
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
if: ${{ runner.os != 'Linux' }} # skip on manylinux2014
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
cache: "yarn"
cache-dependency-path: yarn.lock
if: ${{ runner.os != 'Linux' }} # skip on manylinux2014
################
# Dependencies #
################
################
# JS
- name: Install yarn
run: npm install -g yarn
if: ${{ runner.os != 'Linux' }}
- name: Install js dependencies
run: yarn
env:
PSP_SKIP_EMSDK_INSTALL: 1
################
# Python
- name: Install python dependencies
run: yarn _requires_python
if: ${{ runner.os != 'Linux' }} # skip on manylinux2014
################
# Linux
# NOTE: Not necessary because docker images prebuilt.
# If we remove docker images, we will need to reenable
# this.
# - name: Linux init steps
# run: sudo node tools/perspective-scripts/install_tools.mjs
# if: ${{ runner.os == 'Linux' }}
################
# Mac
- name: Install Boost (MacOS)
run: sudo node tools/perspective-scripts/install_tools.mjs
if: ${{ runner.os == 'macOS' }}
################
# Windows
- name: Windows init steps (vc143)
run: |
vcpkg.exe install boost-thread boost-algorithm boost-filesystem boost-multi-index boost-multiprecision boost-program-options boost-system boost-unordered boost-uuid
vcpkg.exe integrate install
env:
PYTHON_VERSION: ${{ matrix.python-version }}
VCPKG_DEFAULT_TRIPLET: x64-windows
VCPKG_PLATFORM_TOOLSET: v143
if: ${{ matrix.os == 'windows-2022' }}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~ Build Pipelines ~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
########
# Linux
- name: Python Wheel Steps (Linux)
run: yarn _wheel_python --ci
env:
PYTHON_VERSION: ${{ matrix.python-version }}
MANYLINUX: ${{ matrix.container }}
PSP_DOCKER: 1
if: ${{ matrix.os == 'ubuntu-20.04' }}
########
# Macos
- name: Python Build Steps (Macos)
run: |
yarn _wheel_python --ci --macos
yarn _wheel_python --ci --macos --arm
env:
PYTHON_VERSION: ${{ matrix.python-version }}
if: ${{ runner.os == 'macOS' }}
- name: Python SDist Steps
run: |
cd python/perspective/
rm -rf perspective/table/*.{so,dll}
python3 setup.py sdist
env:
PYTHON_VERSION: ${{ matrix.python-version }}
if: ${{ matrix.os == 'macos-11' }}
##########
# Windows
- name: Python Build Steps (Windows vc14.3)
run: yarn _wheel_python --ci
env:
PSP_GENERATOR: Visual Studio 17 2022
PSP_VCPKG_PATH: C:/vcpkg/scripts/buildsystems/vcpkg.cmake
if: ${{ matrix.os == 'windows-2022' }}
####################
# Upload artifacts #
####################
- uses: actions/upload-artifact@v3
with:
name: perspective-python-dist-${{ matrix.os }}-${{ matrix.python-version }}
path: python/perspective/dist/*.whl
if: ${{ runner.os == 'Windows' }}
- uses: actions/upload-artifact@v3
with:
name: perspective-python-dist-${{ matrix.os }}-${{ matrix.python-version }}
path: python/perspective/wheelhouse/*.whl
if: ${{ runner.os != 'Windows' }}
- uses: actions/upload-artifact@v3
with:
name: perspective-python-sdist
path: python/perspective/dist/*.tar.gz
if: ${{ matrix.os == 'macos-11' }}
build_pyodide:
strategy:
matrix:
os: [ubuntu-20.04]
runs-on: ${{ matrix.os }}
needs: [initialize]
env:
PYODIDE_VERSION: 0.23.2
# PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION.
# The appropriate versions can be found in the Pyodide repodata.json
# "info" field, or in Makefile.envs:
# https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2
PYTHON_VERSION: 3.11.2
EMSCRIPTEN_VERSION: 3.1.32
steps:
- name: Checkout perspective
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
- name: set up python
id: setup-python
uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0
with:
python-version: ${{ env.PYTHON_VERSION }}
- uses: mymindstorm/setup-emsdk@ab889da2abbcbb280f91ec4c215d3bb4f3a8f775 # v12
with:
version: ${{ env.EMSCRIPTEN_VERSION }}
actions-cache-folder: emsdk-cache
- name: copy files
run: |
npm install -g yarn
yarn --frozen-lockfile
yarn build_python --setup-only
- name: Linux init steps
run: sudo node tools/perspective-scripts/install_tools.mjs
- name: Install pyodide-build
run: pip install pyodide-build==$PYODIDE_VERSION "pydantic<2"
- name: Build
# Without --exports=pyinit, pyodide-build tries to export all symbols from every .o,
# which causes the em++ linker command line invocation to be so long
# that the command fails.
run: pyodide build python/perspective --exports=pyinit
- name: Upload pyodide wheel
uses: actions/upload-artifact@v3
with:
name: perspective-python-dist-pyodide
path: dist/*.whl
##########################################################################################################################
##########################################################################################################################
###############################################
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~|##########|~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~|##|~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~|##|~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~|##########|~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~|##|~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~|##|~~~~|##|~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~|##########|~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# Stage Five - Test combined JS/Python Assets #
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
test_js_and_python:
###################################################################
# Run JS build first, then use the artifacts in the python builds #
###################################################################
needs: [initialize, build_js, build_python]
strategy:
matrix:
os:
- ubuntu-20.04 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md
python-version:
- 3.9
node-version: [18.x]
# NOTE: don't fail fast as sometimes npm blocks the burst of fetches from GHA
fail-fast: false
##################################
# Setup matrix job configuration #
##################################
runs-on: ${{ matrix.os }}
if: ${{ needs.initialize.outputs.SKIP_CI == 'false' && needs.initialize.outputs.SKIP_PYTHON == 'false' }}
steps:
- name: Checkout
uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
name: perspective-dist
path: packages/perspective/dist/
- uses: actions/download-artifact@v3
with:
name: perspective-jupyterlab-dist
path: packages/perspective-jupyterlab/dist/
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-d3fc-dist
path: packages/perspective-viewer-d3fc/dist
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-datagrid-dist
path: packages/perspective-viewer-datagrid/dist
- uses: actions/download-artifact@v3
with:
name: perspective-workspace-dist
path: packages/perspective-workspace/dist
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-dist
path: rust/perspective-viewer/dist
- uses: actions/download-artifact@v3
with:
name: nbextension-dist
path: python/perspective/perspective/nbextension
- uses: actions/download-artifact@v3
with:
name: labextension-dist
path: python/perspective/perspective/labextension
##########
# Caches #
##########
################
# Yarn Cache
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
- name: Setup yarn cache
uses: actions/cache@v3
id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
with:
path: |
${{ steps.yarn-cache-dir-path.outputs.dir }}
~/.cache/ms-playwright
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
################
# Pip Cache
- name: Setup pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
###############################
# Language and Compiler Setup #
###############################
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
cache: "yarn"
cache-dependency-path: yarn.lock
################
# Dependencies #
################
################
# JS
- name: Install yarn
run: npm install -g yarn
- name: Install js dependencies
run: yarn --frozen-lockfile
env:
PSP_SKIP_EMSDK_INSTALL: 1
- name: Install Playwright Browsers
run: npx playwright install --with-deps
################
# Python
- name: Install python dependencies
run: yarn _requires_python
# Download artifact
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-${{ matrix.os }}-${{ matrix.python-version }}
# Install artifact
- name: Install wheel (system)
run: python -m pip install -U --no-dependencies *manylinux2014*.whl
- name: Install wheel (local)
run: python -m pip install -U --no-dependencies *manylinux2014*.whl --target python/perspective
- name: Check Installed labextensions
run: jupyter labextension list
- name: Run Jupyterlab tests
run: yarn test_js --jupyter
env:
PACKAGE: perspective-jupyterlab
##########################################################################################################################
##########################################################################################################################
########################################
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~|##########|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~|##|~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~|##|~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~|##########|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~|##|~~~~|##|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~|##|~~~~|##|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~|##########|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# Stage Six - Test Python wheel Assets #
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
test_python:
######################################################################
# Run Python build first, then use the artifacts in the python tests #
######################################################################
needs: [initialize, build_python]
strategy:
matrix:
os:
- ubuntu-20.04 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md
- macos-11 # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11-Readme.md
- windows-2022 # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2022-Readme.md
python-version:
- 3.7
- 3.8
- 3.9
- "3.10"
- 3.11
node-version: [18.x]
is-full-run:
- ${{ needs.initialize.outputs.FULL_RUN == 'true' }}
include-windows-run:
- ${{ needs.initialize.outputs.INCLUDE_WINDOWS == 'true' || needs.initialize.outputs.FULL_RUN == 'true' }}
exclude:
##############################################
# Things to exclude if not a full matrix run #
##############################################
# Exclude if skipping windows
- include-windows-run: false
os: windows-2022
# Exclude Mac builds
- is-full-run: false
os: macos-11
# GitHub Actions are broken on OSX, don't test 3.7 wheels
# https://github.com/actions/setup-python/issues/682
- python-version: 3.7
os: macos-11
# Exclude Python 3.7, 3.8, 3.10, 3.11 builds
- is-full-run: false
python-version: 3.7
- is-full-run: false
python-version: 3.8
- is-full-run: false
python-version: "3.10"
- is-full-run: false
python-version: 3.11
# NOTE: don't fail fast as sometimes npm blocks the burst of fetches from GHA
fail-fast: false
##################################
# Setup matrix job configuration #
##################################
runs-on: ${{ matrix.os }}
if: ${{ needs.initialize.outputs.SKIP_CI == 'false' && needs.initialize.outputs.SKIP_PYTHON == 'false' }}
steps:
- name: Checkout
uses: actions/checkout@v3
##########
# Caches #
##########
################
# Yarn Cache
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
- name: Setup yarn cache
uses: actions/cache@v3
id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
with:
path: |
~/.cache/ms-playwright
${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
################
# Pip Cache
- name: Setup pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
###############################
# Language and Compiler Setup #
###############################
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
cache: "yarn"
cache-dependency-path: yarn.lock
################
# Dependencies #
################
################
# JS
- name: Install yarn
run: npm install -g yarn
- name: Install js dependencies
run: yarn
env:
PSP_SKIP_EMSDK_INSTALL: 1
################
# Python
- name: Install python dependencies
run: yarn _requires_python
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~ Build Pipelines ~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#################
# Python - Test #
#################
# Download artifact
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-${{ matrix.os }}-${{ matrix.python-version }}
# Install artifact in-place so tests work as-is
- name: Install wheel (Linux)
run: python -m pip install -U --no-dependencies *manylinux2014*.whl --target python/perspective
if: ${{ runner.os == 'Linux' }}
# Note, on mac we must install the x86 wheel, the arm64 wheel
# would need an arm machine to test
- name: Install wheel (OSX)
run: python -m pip install -U --no-dependencies *x86*.whl --target python/perspective
if: ${{ runner.os == 'macOS' && matrix.python-version != '3.11' }}
- name: Install wheel (OSX 3.11+)
run: python -m pip install -U --no-dependencies *universal*.whl --target python/perspective
if: ${{ runner.os == 'macOS' && matrix.python-version == '3.11' }}
- name: Install wheel (windows)
run: python -m pip install -U --no-dependencies (Get-ChildItem .\*.whl | Select-Object -Expand FullName) --target python/perspective
if: ${{ runner.os == 'Windows' }}
# Run tests
- name: Python Test Steps
run: yarn test_python --debug
##########################################################################################################################
##########################################################################################################################
##########################################
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~|#############|~~~~~~~~~~~#
#~~~~~~~~~~~~~~|#|~~~~~~~/##/~~~~~~~~~~~~#
#~~~~~~~~~~~~~~|#|~~~~~/##/~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~/##/~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~/##/~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~/##/~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~/##/~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# Stage Seven - Test Python sdist Assets #
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
test_python_sdist:
######################################################################
# Run Python build first, then use the artifacts in the python tests #
######################################################################
needs: [initialize, build_python]
strategy:
matrix:
os:
- ubuntu-20.04 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md
python-version:
- 3.9
##################################
# Setup matrix job configuration #
##################################
runs-on: ${{ matrix.os }}
if: ${{ needs.initialize.outputs.FULL_RUN == 'true' }}
steps:
- name: Checkout
uses: actions/checkout@v3
##########
# Caches #
##########
################
# Yarn Cache
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
- name: Setup yarn cache
uses: actions/cache@v3
id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
with:
path: |
~/.cache/ms-playwright
${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
################
# Pip Cache
- name: Setup pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
###############################
# Language and Compiler Setup #
###############################
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
cache: "yarn"
cache-dependency-path: yarn.lock
################
# Dependencies #
################
################
# JS
- name: Install yarn
run: npm install -g yarn
- name: Install js dependencies
run: yarn --frozen-lockfile
env:
PSP_SKIP_EMSDK_INSTALL: 1
################
# Linux
- name: Linux init steps
run: sudo node tools/perspective-scripts/install_tools.mjs
################
# Python
- name: Install python dependencies
run: yarn _requires_python
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~ Build Pipelines ~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# Download sdist
- uses: actions/download-artifact@v3
with:
name: perspective-python-sdist
# Install sdist
- name: Install sdist
run: python -m pip install --no-dependencies perspective*.tar.gz
# Test sdist
- name: Run tests against from-scratch sdist build
run: python -m pytest -vvv --rootdir=python/perspective `python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))"`/perspective/tests --ignore=`python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))"`/perspective/tests/client_mode
##########################################################################################################################
##########################################################################################################################
############################ ##############################
#~~~~~~~~~~~~~~~~~~~~~~~~~~# #~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~|#############|~~~~~# #~~~~~~~~~|##########|~~~~~~~#
#~~~~~~|#|~~~~~~~/##/~~~~~~# #~~~~~~~~~|##|~~~~~~~~~~~~~~~#
#~~~~~~|#|~~~~~/##/~~~~~~~~# #~~~~~~~~~|##|~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~/##/~~~~~~~~~~# #~~~~~~~~~|##########|~~~~~~~#
#~~~~~~~~~~/##/~~~~~~~~~~~~# #~~~~~~~~~~~~~~~~~|##|~~~~~~~#
#~~~~~~~~/##/~~~~~~~~~~~~~~# #~~~~~~~~~|##|~~~~|##|~~~~~~~#
#~~~~~~/##/~~~~~~~~~~~~~~~~# #~~~~~~~~~|##########|~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~# ##### #~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# Stage Seven and a Half # # . # # Benchmarking #
#~~~~~~~~~~~~~~~~~~~~~~~~~~# ##### #~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
benchmark_python:
######################################################################
# Run Python build first, then use the artifacts in the python tests #
######################################################################
needs: [build_python]
strategy:
matrix:
os: [ubuntu-20.04]
python-version: [3.11]
node-version: [18.x]
if: startsWith(github.ref, 'refs/tags/v')
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@v3
##########
# Caches #
##########
################
# Yarn Cache
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
shell: bash
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
- name: Setup yarn cache
uses: actions/cache@v3
id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
with:
path: |
~/.cache/ms-playwright
${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
################
# Pip Cache
- name: Setup pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
if: ${{ needs.initialize.outputs.SKIP_CACHE == 'false' }}
###############################
# Language and Compiler Setup #
###############################
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
cache: "yarn"
cache-dependency-path: yarn.lock
################
# Dependencies #
################
################
# JS
- name: Install yarn
run: npm install -g yarn
- name: Install js dependencies
run: yarn
env:
PSP_SKIP_EMSDK_INSTALL: 1
################
# Python
- name: Install python dependencies
run: yarn _requires_python
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~ Build Pipelines ~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#################
# Python - Test #
#################
# Download artifact
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-${{ matrix.os }}-${{ matrix.python-version }}
# Install artifact in-place so tests work as-is
- name: Install wheel (Linux)
run: python -m pip install -U *manylinux2014*.whl --target python/perspective
if: ${{ runner.os == 'Linux' }}
- name: Run Benchmark
run: yarn bench
shell: bash
env:
PSP_PROJECT: python
- uses: actions/upload-artifact@v3
with:
name: perspective-python-benchmarks
path: tools/perspective-bench/dist/benchmark-python.arrow
##########################################################################################################################
##########################################################################################################################
##########################################
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~/#########\~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~|##|~~~~~|##|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~|##|~~~~~|##|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~|###########|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~|##|~~~~~|##|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~|##|~~~~~|##|~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~\#########/~~~~~~~~~~~~~~#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
# Stage Eight - Publish Artifacts #
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
publish:
needs:
[
test_js_and_python,
test_python,
test_python_sdist,
build_pyodide,
test_js,
benchmark_js,
benchmark_python,
]
if: startsWith(github.ref, 'refs/tags/v')
strategy:
matrix:
os: [ubuntu-20.04]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
# TODO: `actions/upload-artifact` succedds with a warning when the
# path is empty, but `actions/downlaod-artifact` fails, and 3 JS
# packages have no `dist`.
# - uses: actions/download-artifact@v3
# with:
# name: perspective-cli-dist
# path: packages/perspective-cli/dist
- uses: actions/download-artifact@v3
with:
name: perspective-dist
path: packages/perspective/dist/
- uses: actions/download-artifact@v3
with:
name: perspective-jupyterlab-dist
path: packages/perspective-jupyterlab/dist/
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-d3fc-dist
path: packages/perspective-viewer-d3fc/dist
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-datagrid-dist
path: packages/perspective-viewer-datagrid/dist
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-openlayers-dist
path: packages/perspective-viewer-openlayers/dist
- uses: actions/download-artifact@v3
with:
name: perspective-workspace-dist
path: packages/perspective-workspace/dist
- uses: actions/download-artifact@v3
with:
name: perspective-viewer-dist
path: rust/perspective-viewer/dist
# - uses: actions/download-artifact@v3
# with:
# name: perspective-esbuild-plugin-dist
# path: packages/perspective-esbuild-plugin/dist
# - uses: actions/download-artifact@v3
# with:
# name: perspective-webpack-plugin-dist
# path: packages/perspective-webpack-plugin/dist
- uses: actions/download-artifact@v3
with:
name: nbextension-dist
path: python/perspective/perspective/nbextension
- uses: actions/download-artifact@v3
with:
name: labextension-dist
path: python/perspective/perspective/labextension
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-ubuntu-20.04-3.7
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-ubuntu-20.04-3.8
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-ubuntu-20.04-3.9
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-ubuntu-20.04-3.10
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-ubuntu-20.04-3.11
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-macos-11-3.7
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-macos-11-3.8
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-macos-11-3.9
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-macos-11-3.10
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-macos-11-3.11
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-windows-2022-3.7
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-windows-2022-3.8
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-windows-2022-3.9
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-windows-2022-3.10
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-windows-2022-3.11
- uses: actions/download-artifact@v3
with:
name: perspective-python-dist-pyodide
- uses: actions/download-artifact@v3
with:
name: perspective-python-sdist
- uses: actions/download-artifact@v3
with:
name: perspective-js-benchmarks
- uses: actions/download-artifact@v3
with:
name: perspective-python-benchmarks
- name: NPM pack
run: |
npm pack \
--workspace=@finos/perspective \
--workspace=@finos/perspective-viewer \
--workspace=@finos/perspective-viewer-datagrid \
--workspace=@finos/perspective-viewer-d3fc \
--workspace=@finos/perspective-viewer-openlayers \
--workspace=@finos/perspective-workspace \
--workspace=@finos/perspective-cli \
--workspace=@finos/perspective-webpack-plugin \
--workspace=@finos/perspective-esbuild-plugin \
--workspace=@finos/perspective-jupyterlab
- name: Debug
run: ls -lah
- name: Release wheels
uses: softprops/action-gh-release@v1
with:
draft: true
generate_release_notes: true
files: |
*.whl
*.tar.gz
*.tgz
*.arrow
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}