Add requirements regenerator script and action

.github/workflows/build.yml

@@ -309,6 +309,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             ################
             # Linux
@@ -508,6 +510,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
             #~~~~~~~~~ Build Pipelines ~~~~~~~~~#
@@ -641,6 +645,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             ################
             # Rust
@@ -1029,6 +1035,8 @@ jobs:
             - name: Install python dependencies
               run: yarn _requires_python
               if: ${{ runner.os != 'Linux' }} # skip on manylinux2014
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             ################
             # Linux
@@ -1323,6 +1331,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             # Download artifact
             - uses: actions/download-artifact@v3
@@ -1491,6 +1501,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
             #~~~~~~~~~ Build Pipelines ~~~~~~~~~#
@@ -1636,6 +1648,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
             #~~~~~~~~~ Build Pipelines ~~~~~~~~~#
@@ -1751,6 +1765,8 @@ jobs:
             # Python
             - name: Install python dependencies
               run: yarn _requires_python
+              env:
+                  PSP_FROZEN_REQUIREMENTS: "1"
 
             #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
             #~~~~~~~~~ Build Pipelines ~~~~~~~~~#

.github/workflows/python-deps.yml

@@ -0,0 +1,91 @@
+name: Regenerate Python Dependencies
+
+on:
+    workflow_dispatch:
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+    cancel-in-progress: true
+
+permissions:
+    contents: write
+
+jobs:
+    generate:
+        strategy:
+            matrix:
+                python-version:
+                    - "3.7"
+                    - "3.8"
+                    - "3.9"
+                    - "3.10"
+                    - "3.11"
+                node-version: [18.x]
+            fail-fast: false
+
+        runs-on: ubuntu-20.04
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v3
+
+            - name: Use Node.js ${{ matrix.node-version }}
+              uses: actions/setup-node@v3
+              with:
+                  node-version: ${{ matrix.node-version }}
+                  cache: "yarn"
+                  cache-dependency-path: yarn.lock
+
+            - name: Install js dependencies
+              run: yarn
+              env:
+                  PSP_SKIP_EMSDK_INSTALL: 1
+
+            - name: Set up Python ${{ matrix.python-version }}
+              uses: actions/setup-python@v4
+              with:
+                  python-version: ${{ matrix.python-version }}
+                  cache: "pip"
+
+            - name: Install python dependencies
+              run: yarn _requires_python
+              env:
+                  PSP_GENERATE_FROZEN_REQUIREMENTS: "1"
+
+            - name: Push changes to well-known branch name
+              run: |
+                  git config --global user.email "[email protected]"
+                  git config --global user.name "Tim Paine (Automated)"
+                  git checkout -b tkp/update-frozen-python-${{ matrix.python-version }}-dependencies
+                  git add python/perspective/ci-requirements/
+                  git commit -m "Update frozen python dependencies for ${{ matrix.python-version }}"
+                  git push origin tkp/update-frozen-python-${{ matrix.python-version }}-dependencies -f --no-verify
+
+    prepare:
+        needs: [generate]
+        runs-on: ubuntu-20.04
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v3
+
+            - name: Collect changes and meld them into a single branch
+              run: |
+                  git config --global user.email "[email protected]"
+                  git config --global user.name "Tim Paine (Automated)"
+                  git checkout -b tkp/update-frozen-python-dependencies
+                  git fetch --all
+
+                  git merge origin/tkp/update-frozen-python-3.7-dependencies --squash --no-edit
+                  git merge origin/tkp/update-frozen-python-3.8-dependencies --squash --no-edit
+                  git merge origin/tkp/update-frozen-python-3.9-dependencies --squash --no-edit
+                  git merge origin/tkp/update-frozen-python-3.10-dependencies --squash --no-edit
+                  git merge origin/tkp/update-frozen-python-3.11-dependencies --squash --no-edit
+                  git commit -m "Update frozen python dependencies"
+
+                  git push origin tkp/update-frozen-python-dependencies -f --no-verify
+
+                  git push origin --delete tkp/update-frozen-python-3.7-dependencies
+                  git push origin --delete tkp/update-frozen-python-3.8-dependencies
+                  git push origin --delete tkp/update-frozen-python-3.9-dependencies
+                  git push origin --delete tkp/update-frozen-python-3.10-dependencies
+                  git push origin --delete tkp/update-frozen-python-3.11-dependencies

.gitignore

@@ -132,6 +132,9 @@ website/i18n/*
 !website/i18n/en.json
 website/static/css/material-dark.css
 website/static/css/pro-dark.css
+docs/static/features
+docs/.docusaurus
+docs/static/blocks
 
 # test artifacts
 .ipynb_checkpoints
@@ -143,6 +146,10 @@ coverage
 screenshots/
 junit.xml
 results.debug.json
+test-results/
+playwright-report/
+playwright/.cache/
+.cache
 
 # CPP Compile
 /src/include/boost
@@ -204,21 +211,10 @@ vcpkg
 # jupyterlab test artifacts
 packages/perspective-jupyterlab/test/config/jupyter/lab
 packages/perspective-jupyterlab/test/config/jupyter/migrated
-docs/static/features
-results.debug.json
-
-tools/perspective-build/lib
-docs/.docusaurus
-packages/perspective-esbuild-plugin/lib
-docs/static/blocks
-.vscode/c_cpp_properties.json
-test-results/
-playwright-report/
-playwright/.cache/
 
+# Build envs / virtualenvs / other local
 .pyodide-xbuildenv
 benchmark_venv
-
 venv/
+.venvs/
 testenv
-.cache

python/perspective/MANIFEST.in

@@ -31,6 +31,9 @@ prune bench
 # Don't ship scripts
 prune scripts
 
+# Don't ship test artifacts
+prune ci-requirements
+
 # Patterns to exclude from any directory
 global-exclude *~
 global-exclude *.pyc

tools/perspective-scripts/_requires_python.mjs

@@ -13,28 +13,17 @@
 import {
     default as sh,
     python_version,
-    copy_files_to_python_folder,
-    py_requirements,
+    generate_py_requirements,
+    install_py_requirements_dynamic,
+    install_py_requirements_frozen,
 } from "./sh_perspective.mjs";
 
 let PYTHON = sh(python_version());
 
-if (process.env.PSP_OLD_SHITTY_INSTALL_METHOD) {
-    const requires_script = `import distutils.core; setup = distutils.core.run_setup('python/perspective/setup.py'); print(' '.join(['"' + requirement + '"' for requirement in setup.extras_require['dev']]))`;
-
-    // copy build/config files into python folder
-    copy_files_to_python_folder();
-
-    // install build meta  deps, this is a necessary evil to keep the setup.py clean
-    sh`${PYTHON} -m pip install  jupyter_packaging==0.12.3`.runSync();
-    const requirements = await sh`${PYTHON} -c ${requires_script}`.execSync();
-    if (requirements.trim().length > 0) {
-        console.log(`Installing: ${requirements}`);
-        sh`${PYTHON} -m pip install -U ${sh(requirements)}`.log().runSync();
-    } else {
-        console.log("Nothing to install");
-    }
+if (process.env.PSP_GENERATE_FROZEN_REQUIREMENTS) {
+    generate_py_requirements(PYTHON);
+} else if (process.env.PSP_FROZEN_REQUIREMENTS) {
+    install_py_requirements_frozen(PYTHON);
 } else {
-    sh`${PYTHON} -m pip install -r ${py_requirements()}`.runSync();
-    sh`${PYTHON} -m pip install -U jupyter_packaging==0.12.3`.runSync();
+    install_py_requirements_dynamic(PYTHON);
 }

tools/perspective-scripts/sh_perspective.mjs

@@ -253,13 +253,54 @@ export const copy_files_to_python_folder = (link_files) => {
     }
 };
 
-export function py_requirements() {
-    const version = sh`python3 --version`
+function get_python_version_for_requirements() {
+    return sh`python3 --version`
         .execSync()
         .replace("Python ", "")
         .replace(".", "")
         .replace(/\..*?$/m, "");
-    return `python/perspective/requirements-${version}.txt`;
+}
+
+export function py_requirements() {
+    return `python/perspective/ci-requirements/requirements-${get_python_version_for_requirements()}.txt`;
+}
+
+export function generate_py_requirements(python) {
+    const version = get_python_version_for_requirements();
+    if (!fs.existsSync(".venvs")) {
+        fs.mkdirSync(".venvs");
+    }
+
+    if (fs.existsSync(`.venvs/${version}`)) {
+        rimraf.sync(`.venvs/${version}`);
+    }
+    sh`${python} -m venv .venvs/${version}`.runSync();
+    sh`. .venvs/${version}/bin/activate`;
+    process.env.PATH = `.venvs/${version}/bin:${process.env.PATH}`;
+    install_py_requirements_dynamic(`.venvs/${version}/bin/python`);
+    sh`.venvs/${version}/bin/python -m pip freeze > python/perspective/ci-requirements/requirements-${version}.txt`.runSync();
+}
+
+export function install_py_requirements_dynamic(python) {
+    const requires_script = `import distutils.core; setup = distutils.core.run_setup('python/perspective/setup.py'); print(' '.join(['"' + requirement + '"' for requirement in setup.extras_require['dev']]))`;
+
+    // copy build/config files into python folder
+    copy_files_to_python_folder();
+
+    // install build meta deps, this is a necessary evil to keep the setup.py clean
+    sh`${python} -m pip install  jupyter_packaging==0.12.3`.runSync();
+    const requirements = sh`${python} -c ${requires_script}`.execSync();
+    if (requirements.trim().length > 0) {
+        console.log(`Installing: ${requirements}`);
+        sh`${python} -m pip install -U ${sh(requirements)}`.log().runSync();
+    } else {
+        console.log("Nothing to install");
+    }
+}
+
+export function install_py_requirements_frozen(python) {
+    sh`${python} -m pip install -r ${py_requirements()}`.runSync();
+    sh`${python} -m pip install -U jupyter_packaging==0.12.3`.runSync();
 }
 
 /**
@@ -342,8 +383,8 @@ sh.docker = function docker(...args) {
         ${env_vars} \
         -v${CWD}:/usr/src/app/perspective \
         -w /usr/src/app/perspective --shm-size=2g -u root \
-        --cpus="${CPUS}.0" 
-        ${IMAGE} 
+        --cpus="${CPUS}.0"
+        ${IMAGE}
         bash -c ${sh(...args).toString()}`;
 };