From 36406ef86b183284b2c328003294ab2e6290315e Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Sat, 4 Nov 2023 16:16:29 -0400 Subject: [PATCH] Add requirements regenerator script and action --- .github/workflows/build.yml | 16 ++++ .github/workflows/python-deps.yml | 91 +++++++++++++++++++ .gitignore | 22 ++--- python/perspective/MANIFEST.in | 3 + .../requirements-310.txt | 0 .../requirements-311.txt | 0 .../{ => ci-requirements}/requirements-37.txt | 0 .../{ => ci-requirements}/requirements-38.txt | 0 .../{ => ci-requirements}/requirements-39.txt | 0 .../perspective-scripts/_requires_python.mjs | 27 ++---- tools/perspective-scripts/sh_perspective.mjs | 51 ++++++++++- 11 files changed, 173 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/python-deps.yml rename python/perspective/{ => ci-requirements}/requirements-310.txt (100%) rename python/perspective/{ => ci-requirements}/requirements-311.txt (100%) rename python/perspective/{ => ci-requirements}/requirements-37.txt (100%) rename python/perspective/{ => ci-requirements}/requirements-38.txt (100%) rename python/perspective/{ => ci-requirements}/requirements-39.txt (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1aee6e8e75..6854db561b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -309,6 +309,8 @@ jobs: # Python - name: Install python dependencies run: yarn _requires_python + env: + PSP_FROZEN_REQUIREMENTS: "1" ################ # Linux @@ -508,6 +510,8 @@ jobs: # Python - name: Install python dependencies run: yarn _requires_python + env: + PSP_FROZEN_REQUIREMENTS: "1" #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# #~~~~~~~~~ Build Pipelines ~~~~~~~~~# @@ -641,6 +645,8 @@ jobs: # Python - name: Install python dependencies run: yarn _requires_python + env: + PSP_FROZEN_REQUIREMENTS: "1" ################ # Rust @@ -1029,6 +1035,8 @@ jobs: - name: Install python dependencies run: yarn _requires_python if: ${{ runner.os != 'Linux' }} # skip on manylinux2014 + env: + PSP_FROZEN_REQUIREMENTS: "1" ################ # Linux @@ -1323,6 +1331,8 @@ jobs: # Python - name: Install python dependencies run: yarn _requires_python + env: + PSP_FROZEN_REQUIREMENTS: "1" # Download artifact - uses: actions/download-artifact@v3 @@ -1491,6 +1501,8 @@ jobs: # Python - name: Install python dependencies run: yarn _requires_python + env: + PSP_FROZEN_REQUIREMENTS: "1" #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# #~~~~~~~~~ Build Pipelines ~~~~~~~~~# @@ -1636,6 +1648,8 @@ jobs: # Python - name: Install python dependencies run: yarn _requires_python + env: + PSP_FROZEN_REQUIREMENTS: "1" #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# #~~~~~~~~~ Build Pipelines ~~~~~~~~~# @@ -1751,6 +1765,8 @@ jobs: # Python - name: Install python dependencies run: yarn _requires_python + env: + PSP_FROZEN_REQUIREMENTS: "1" #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# #~~~~~~~~~ Build Pipelines ~~~~~~~~~# diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml new file mode 100644 index 0000000000..1a2b1474a1 --- /dev/null +++ b/.github/workflows/python-deps.yml @@ -0,0 +1,91 @@ +name: Regenerate Python Dependencies + +on: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: write + +jobs: + generate: + strategy: + matrix: + python-version: + - "3.7" + - "3.8" + - "3.9" + - "3.10" + - "3.11" + node-version: [18.x] + fail-fast: false + + runs-on: ubuntu-20.04 + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: "yarn" + cache-dependency-path: yarn.lock + + - name: Install js dependencies + run: yarn + env: + PSP_SKIP_EMSDK_INSTALL: 1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install python dependencies + run: yarn _requires_python + env: + PSP_GENERATE_FROZEN_REQUIREMENTS: "1" + + - name: Push changes to well-known branch name + run: | + git config --global user.email "3105306+timkpaine@users.noreply.github.com" + git config --global user.name "Tim Paine (Automated)" + git checkout -b tkp/update-frozen-python-${{ matrix.python-version }}-dependencies + git add python/perspective/ci-requirements/ + git commit -m "Update frozen python dependencies for ${{ matrix.python-version }}" + git push origin tkp/update-frozen-python-${{ matrix.python-version }}-dependencies -f --no-verify + + prepare: + needs: [generate] + runs-on: ubuntu-20.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Collect changes and meld them into a single branch + run: | + git config --global user.email "3105306+timkpaine@users.noreply.github.com" + git config --global user.name "Tim Paine (Automated)" + git checkout -b tkp/update-frozen-python-dependencies + git fetch --all + + git merge origin/tkp/update-frozen-python-3.7-dependencies --squash --no-edit + git merge origin/tkp/update-frozen-python-3.8-dependencies --squash --no-edit + git merge origin/tkp/update-frozen-python-3.9-dependencies --squash --no-edit + git merge origin/tkp/update-frozen-python-3.10-dependencies --squash --no-edit + git merge origin/tkp/update-frozen-python-3.11-dependencies --squash --no-edit + git commit -m "Update frozen python dependencies" + + git push origin tkp/update-frozen-python-dependencies -f --no-verify + + git push origin --delete tkp/update-frozen-python-3.7-dependencies + git push origin --delete tkp/update-frozen-python-3.8-dependencies + git push origin --delete tkp/update-frozen-python-3.9-dependencies + git push origin --delete tkp/update-frozen-python-3.10-dependencies + git push origin --delete tkp/update-frozen-python-3.11-dependencies diff --git a/.gitignore b/.gitignore index 578be485e1..df02de88d2 100644 --- a/.gitignore +++ b/.gitignore @@ -132,6 +132,9 @@ website/i18n/* !website/i18n/en.json website/static/css/material-dark.css website/static/css/pro-dark.css +docs/static/features +docs/.docusaurus +docs/static/blocks # test artifacts .ipynb_checkpoints @@ -143,6 +146,10 @@ coverage screenshots/ junit.xml results.debug.json +test-results/ +playwright-report/ +playwright/.cache/ +.cache # CPP Compile /src/include/boost @@ -204,21 +211,10 @@ vcpkg # jupyterlab test artifacts packages/perspective-jupyterlab/test/config/jupyter/lab packages/perspective-jupyterlab/test/config/jupyter/migrated -docs/static/features -results.debug.json - -tools/perspective-build/lib -docs/.docusaurus -packages/perspective-esbuild-plugin/lib -docs/static/blocks -.vscode/c_cpp_properties.json -test-results/ -playwright-report/ -playwright/.cache/ +# Build envs / virtualenvs / other local .pyodide-xbuildenv benchmark_venv - venv/ +.venvs/ testenv -.cache diff --git a/python/perspective/MANIFEST.in b/python/perspective/MANIFEST.in index e746a0fd03..57629b9666 100644 --- a/python/perspective/MANIFEST.in +++ b/python/perspective/MANIFEST.in @@ -31,6 +31,9 @@ prune bench # Don't ship scripts prune scripts +# Don't ship test artifacts +prune ci-requirements + # Patterns to exclude from any directory global-exclude *~ global-exclude *.pyc diff --git a/python/perspective/requirements-310.txt b/python/perspective/ci-requirements/requirements-310.txt similarity index 100% rename from python/perspective/requirements-310.txt rename to python/perspective/ci-requirements/requirements-310.txt diff --git a/python/perspective/requirements-311.txt b/python/perspective/ci-requirements/requirements-311.txt similarity index 100% rename from python/perspective/requirements-311.txt rename to python/perspective/ci-requirements/requirements-311.txt diff --git a/python/perspective/requirements-37.txt b/python/perspective/ci-requirements/requirements-37.txt similarity index 100% rename from python/perspective/requirements-37.txt rename to python/perspective/ci-requirements/requirements-37.txt diff --git a/python/perspective/requirements-38.txt b/python/perspective/ci-requirements/requirements-38.txt similarity index 100% rename from python/perspective/requirements-38.txt rename to python/perspective/ci-requirements/requirements-38.txt diff --git a/python/perspective/requirements-39.txt b/python/perspective/ci-requirements/requirements-39.txt similarity index 100% rename from python/perspective/requirements-39.txt rename to python/perspective/ci-requirements/requirements-39.txt diff --git a/tools/perspective-scripts/_requires_python.mjs b/tools/perspective-scripts/_requires_python.mjs index 777336c913..8920edeb6a 100644 --- a/tools/perspective-scripts/_requires_python.mjs +++ b/tools/perspective-scripts/_requires_python.mjs @@ -13,28 +13,17 @@ import { default as sh, python_version, - copy_files_to_python_folder, - py_requirements, + generate_py_requirements, + install_py_requirements_dynamic, + install_py_requirements_frozen, } from "./sh_perspective.mjs"; let PYTHON = sh(python_version()); -if (process.env.PSP_OLD_SHITTY_INSTALL_METHOD) { - const requires_script = `import distutils.core; setup = distutils.core.run_setup('python/perspective/setup.py'); print(' '.join(['"' + requirement + '"' for requirement in setup.extras_require['dev']]))`; - - // copy build/config files into python folder - copy_files_to_python_folder(); - - // install build meta deps, this is a necessary evil to keep the setup.py clean - sh`${PYTHON} -m pip install jupyter_packaging==0.12.3`.runSync(); - const requirements = await sh`${PYTHON} -c ${requires_script}`.execSync(); - if (requirements.trim().length > 0) { - console.log(`Installing: ${requirements}`); - sh`${PYTHON} -m pip install -U ${sh(requirements)}`.log().runSync(); - } else { - console.log("Nothing to install"); - } +if (process.env.PSP_GENERATE_FROZEN_REQUIREMENTS) { + generate_py_requirements(PYTHON); +} else if (process.env.PSP_FROZEN_REQUIREMENTS) { + install_py_requirements_frozen(PYTHON); } else { - sh`${PYTHON} -m pip install -r ${py_requirements()}`.runSync(); - sh`${PYTHON} -m pip install -U jupyter_packaging==0.12.3`.runSync(); + install_py_requirements_dynamic(PYTHON); } diff --git a/tools/perspective-scripts/sh_perspective.mjs b/tools/perspective-scripts/sh_perspective.mjs index 77c127a985..67c9211036 100644 --- a/tools/perspective-scripts/sh_perspective.mjs +++ b/tools/perspective-scripts/sh_perspective.mjs @@ -253,13 +253,54 @@ export const copy_files_to_python_folder = (link_files) => { } }; -export function py_requirements() { - const version = sh`python3 --version` +function get_python_version_for_requirements() { + return sh`python3 --version` .execSync() .replace("Python ", "") .replace(".", "") .replace(/\..*?$/m, ""); - return `python/perspective/requirements-${version}.txt`; +} + +export function py_requirements() { + return `python/perspective/ci-requirements/requirements-${get_python_version_for_requirements()}.txt`; +} + +export function generate_py_requirements(python) { + const version = get_python_version_for_requirements(); + if (!fs.existsSync(".venvs")) { + fs.mkdirSync(".venvs"); + } + + if (fs.existsSync(`.venvs/${version}`)) { + rimraf.sync(`.venvs/${version}`); + } + sh`${python} -m venv .venvs/${version}`.runSync(); + sh`. .venvs/${version}/bin/activate`; + process.env.PATH = `.venvs/${version}/bin:${process.env.PATH}`; + install_py_requirements_dynamic(`.venvs/${version}/bin/python`); + sh`.venvs/${version}/bin/python -m pip freeze > python/perspective/ci-requirements/requirements-${version}.txt`.runSync(); +} + +export function install_py_requirements_dynamic(python) { + const requires_script = `import distutils.core; setup = distutils.core.run_setup('python/perspective/setup.py'); print(' '.join(['"' + requirement + '"' for requirement in setup.extras_require['dev']]))`; + + // copy build/config files into python folder + copy_files_to_python_folder(); + + // install build meta deps, this is a necessary evil to keep the setup.py clean + sh`${python} -m pip install jupyter_packaging==0.12.3`.runSync(); + const requirements = sh`${python} -c ${requires_script}`.execSync(); + if (requirements.trim().length > 0) { + console.log(`Installing: ${requirements}`); + sh`${python} -m pip install -U ${sh(requirements)}`.log().runSync(); + } else { + console.log("Nothing to install"); + } +} + +export function install_py_requirements_frozen(python) { + sh`${python} -m pip install -r ${py_requirements()}`.runSync(); + sh`${python} -m pip install -U jupyter_packaging==0.12.3`.runSync(); } /** @@ -342,8 +383,8 @@ sh.docker = function docker(...args) { ${env_vars} \ -v${CWD}:/usr/src/app/perspective \ -w /usr/src/app/perspective --shm-size=2g -u root \ - --cpus="${CPUS}.0" - ${IMAGE} + --cpus="${CPUS}.0" + ${IMAGE} bash -c ${sh(...args).toString()}`; };