python 3.10

.github/scripts/build-linux.sh

@@ -1,33 +1,28 @@
 #!/bin/bash
 set -e -x
 
-bash --version
-
 cd /io
 
 source .github/scripts/retry.sh
 
 # List python versions
 ls /opt/python
 
-if [ $PYTHON_VERSION == "3.5" ]; then
-    PYBIN="/opt/python/cp35-cp35m/bin"
-elif [ $PYTHON_VERSION == "3.6" ]; then
+if [ $PYTHON_VERSION == "3.6" ]; then
     PYBIN="/opt/python/cp36-cp36m/bin"
 elif [ $PYTHON_VERSION == "3.7" ]; then
     PYBIN="/opt/python/cp37-cp37m/bin"
 elif [ $PYTHON_VERSION == "3.8" ]; then
     PYBIN="/opt/python/cp38-cp38/bin"
 elif [ $PYTHON_VERSION == "3.9" ]; then
     PYBIN="/opt/python/cp39-cp39/bin"
+elif [ $PYTHON_VERSION == "3.10" ]; then
+    PYBIN="/opt/python/cp310-cp310/bin"
 else
     echo "Unsupported Python version $PYTHON_VERSION"
     exit 1
 fi
 
-# Install build tools
-retry yum install -y cmake
-
 # Install liblensfun
 pushd external/lensfun
 cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=off -DINSTALL_HELPER_SCRIPTS=off .
@@ -61,7 +56,7 @@ retry ${PYBIN}/pip install -r dev-requirements.txt
 retry ${PYBIN}/pip install -U numpy # scipy should trigger an update, but that doesn't happen
 
 pushd $HOME
-${PYBIN}/nosetests --verbosity=3 --nocapture /io/test
+${PYBIN}/pytest --verbosity=3 -s /io/test
 popd
 
 # Move wheel to dist/ folder for easier deployment

.github/scripts/build-macos.sh

@@ -3,6 +3,8 @@ set -e -x
 
 source .github/scripts/retry.sh
 
+CHECK_SHA256=.github/scripts/check_sha256.sh
+
 brew install pkg-config meson
 
 # General note:
@@ -16,17 +18,6 @@ export MACOSX_DEPLOYMENT_TARGET=$MACOS_MIN_VERSION
 # The Python variant to install, see exception below.
 export PYTHON_INSTALLER_MACOS_VERSION=$MACOS_MIN_VERSION
 
-# Work-around issue building on newer XCode versions.
-# https://github.com/pandas-dev/pandas/issues/23424#issuecomment-446393981
-if [ $PYTHON_VERSION == "3.5" ]; then
-    # No 10.9 installer available, use 10.6.
-    # The resulting wheel platform tags still have 10.6 (=target of Python itself),
-    # even though technically the wheel should only be run on 10.9 upwards.
-    # This is fixed manually below by renaming the wheel.
-    # See https://github.com/pypa/wheel/issues/312.
-    export PYTHON_INSTALLER_MACOS_VERSION=10.6
-fi
-
 # Install Python
 # Note: The GitHub Actions supplied Python versions are not used
 # as they are built without MACOSX_DEPLOYMENT_TARGET/-mmacosx-version-min
@@ -57,19 +48,24 @@ pip freeze
 # See https://discourse.brew.sh/t/it-is-possible-to-build-packages-that-are-compatible-with-older-macos-versions/4421
 
 LIB_INSTALL_PREFIX=$(pwd)/external/libs
+export CMAKE_PREFIX_PATH=$LIB_INSTALL_PREFIX
 export PKG_CONFIG_PATH=$LIB_INSTALL_PREFIX/lib/pkgconfig
 export LIBRARY_PATH=$LIB_INSTALL_PREFIX/lib
 export PATH=$LIB_INSTALL_PREFIX/bin:$PATH
 
 # Install libffi (glib dependency)
-curl -L --retry 3 https://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz | tar xz
+curl -L --retry 3 -o libffi.tar.gz https://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz
+$CHECK_SHA256 libffi.tar.gz d06ebb8e1d9a22d19e38d63fdb83954253f39bedc5d46232a05645685722ca37
+tar xzf libffi.tar.gz
 pushd libffi-3.2.1
 ./configure --prefix=$LIB_INSTALL_PREFIX --disable-debug
 make install -j
 popd
 
 # Install gettext (glib dependency)
-curl -L --retry 3 https://ftp.gnu.org/gnu/gettext/gettext-0.20.1.tar.xz | tar xz
+curl -L --retry 3 -o gettext.tar.xz https://ftp.gnu.org/gnu/gettext/gettext-0.20.1.tar.xz
+$CHECK_SHA256 gettext.tar.xz 53f02fbbec9e798b0faaf7c73272f83608e835c6288dd58be6c9bb54624a3800
+tar xzf gettext.tar.xz
 pushd gettext-0.20.1
 ./configure --prefix=$LIB_INSTALL_PREFIX \
     --disable-debug \
@@ -80,12 +76,13 @@ make install
 popd
 
 # Install glib (lensfun dependency)
-curl -L --retry 3 https://download.gnome.org/sources/glib/2.69/glib-2.69.2.tar.xz | tar xz
+curl -L --retry 3 -o glib.tar.xz https://download.gnome.org/sources/glib/2.69/glib-2.69.2.tar.xz
+$CHECK_SHA256 glib.tar.xz a62249e35a8635175a697b3215f1df2b89e0fbb4adb520dcbe21a3ae1ebb8882
+tar xzf glib.tar.xz
 pushd glib-2.69.2
 mkdir build
 cd build
 meson --prefix=$LIB_INSTALL_PREFIX \
-  -Dinternal_pcre=true \
   -Dselinux=disabled \
   -Ddtrace=false \
   -Dman=false \
@@ -105,15 +102,8 @@ export LDFLAGS=$CFLAGS
 export ARCHFLAGS=$CFLAGS
 
 # Build wheel
-export CMAKE_PREFIX_PATH=$LIB_INSTALL_PREFIX
 python setup.py bdist_wheel
 
-# Fix wheel platform tag, see above for details.
-if [ $PYTHON_VERSION == "3.5" ]; then
-    filename=$(ls dist/*.whl)
-    mv -v "$filename" "${filename/macosx_10_6_intel/macosx_10_9_x86_64}"
-fi
-
 # List direct and indirect library dependencies
 mkdir tmp_wheel
 pushd tmp_wheel
@@ -155,5 +145,5 @@ retry pip install -r dev-requirements.txt
 rm -rf $LIB_INSTALL_PREFIX
 mkdir tmp_for_test
 pushd tmp_for_test
-nosetests --verbosity=3 --nocapture ../test
+pytest --verbosity=3 -s ../test
 popd

.github/scripts/build-windows.ps1

@@ -1,9 +1,5 @@
 $ErrorActionPreference = 'Stop'
 
-function not-exist { -not (Test-Path $args) }
-Set-Alias !exists not-exist -Option "Constant, AllScope"
-Set-Alias exists Test-Path -Option "Constant, AllScope"
-
 function exec {
     [CmdletBinding()]
     param([Parameter(Position=0,Mandatory=1)][scriptblock]$cmd)
@@ -17,52 +13,84 @@ function exec {
     }
 }
 
-function Init-VS {
+function Initialize-Python {
+    if ($env:USE_CONDA -eq 1) {
+        $env:CONDA_ROOT = $pwd.Path + "\external\miniconda_$env:PYTHON_ARCH"
+        & .\.github\scripts\install-miniconda.ps1
+        & $env:CONDA_ROOT\shell\condabin\conda-hook.ps1
+        exec { conda update --yes -n base -c defaults conda }
+    }
+    # Check Python version/arch
+    exec { python -c "import platform; assert platform.python_version().startswith('$env:PYTHON_VERSION')" }
+    exec { python -c "import struct; assert struct.calcsize('P') * 8 == $env:PYTHON_ARCH" }
+}
+
+function Create-VEnv {
+    [CmdletBinding()]
+    param([Parameter(Position=0,Mandatory=1)][string]$name)
+    if ($env:USE_CONDA -eq 1) {
+        exec { conda create --yes --name $name -c defaults --strict-channel-priority python=$env:PYTHON_VERSION --force }
+    } else {
+        exec { python -m venv env\$name }
+    }
+}
+
+function Enter-VEnv {
+    [CmdletBinding()]
+    param([Parameter(Position=0,Mandatory=1)][string]$name)
+    if ($env:USE_CONDA -eq 1) {
+        conda activate $name
+    } else {
+        & .\env\$name\scripts\activate
+    }
+}
+
+function Create-And-Enter-VEnv {
+    [CmdletBinding()]
+    param([Parameter(Position=0,Mandatory=1)][string]$name)
+    Create-VEnv $name
+    Enter-VEnv $name
+}
+
+function Exit-VEnv {
+    if ($env:USE_CONDA -eq 1) {
+        conda deactivate
+    } else {
+        deactivate
+    }
+}
+
+function Initialize-VS {
     # https://wiki.python.org/moin/WindowsCompilers
     # setuptools automatically selects the right compiler for building
     # the extension module. The following is mostly for building any
-    # dependencies like liblensfun.
+    # native dependencies, here via CMake.
     # https://docs.microsoft.com/en-us/cpp/build/building-on-the-command-line
     # https://docs.microsoft.com/en-us/cpp/porting/binary-compat-2015-2017
 
-    $VS2017_ROOT = "C:\Program Files (x86)\Microsoft Visual Studio\2017"
-    $VS2019_ROOT = "C:\Program Files (x86)\Microsoft Visual Studio\2019"
+    $VS_ROOT = "C:\Program Files (x86)\Microsoft Visual Studio"
+    $VS_VERSIONS = @("2017", "2019")
+    $VS_EDITIONS = @("Enterprise", "Professional", "Community")
+    $VS_INIT_CMD_SUFFIX = "Common7\Tools\vsdevcmd.bat"
 
     $VS_ARCH = if ($env:PYTHON_ARCH -eq '32') { 'x86' } else { 'x64' }
-
-    $PYTHON_VERSION_TUPLE = $env:PYTHON_VERSION.split('.')
-    $PYTHON_VERSION_MAJOR = $PYTHON_VERSION_TUPLE[0]
-    $PYTHON_VERSION_MINOR = $PYTHON_VERSION_TUPLE[1]
-
-    if ($PYTHON_VERSION_MAJOR -eq '3') {
-        if ($PYTHON_VERSION_MINOR -le '4') {
-            throw ("Python <= 3.4 unsupported: $env:PYTHON_VERSION")
-        }
-        if (exists $VS2017_ROOT) {
-            $VS_VERSION = "2017"
-            if (exists "$VS2017_ROOT\Enterprise") {
-                $VS_ROOT = "$VS2017_ROOT\Enterprise"
-            } else {
-                $VS_ROOT = "$VS2017_ROOT\Community"
+    $VS_INIT_ARGS = "-arch=$VS_ARCH -no_logo"
+
+    $found = $false
+    :outer foreach ($version in $VS_VERSIONS) {
+        foreach ($edition in $VS_EDITIONS) {
+            $VS_INIT_CMD = "$VS_ROOT\$version\$edition\$VS_INIT_CMD_SUFFIX"
+            if (Test-Path $VS_INIT_CMD) {
+                $found = $true
+                break outer
             }
-            $VS_INIT_CMD = "$VS_ROOT\Common7\Tools\vsdevcmd.bat"
-            $VS_INIT_ARGS = "-arch=$VS_ARCH -no_logo"
-        } elseif (exists $VS2019_ROOT) {
-            $VS_VERSION = "2019"
-            if (exists "$VS2019_ROOT\Enterprise") {
-                $VS_ROOT = "$VS2019_ROOT\Enterprise"
-            } else {
-                $VS_ROOT = "$VS2019_ROOT\Community"
-            }
-            $VS_INIT_CMD = "$VS_ROOT\Common7\Tools\vsdevcmd.bat"
-            $VS_INIT_ARGS = "-arch=$VS_ARCH -no_logo"
-        } else {
-            throw ("No suitable Visual Studio installation found")
         }
-    } else {
-        throw ("Unsupported Python version: $PYTHON_VERSION_MAJOR")
     }
-    Write-Host "Configuring VS$VS_VERSION for Python $env:PYTHON_VERSION on a $env:PYTHON_ARCH bit architecture"
+
+    if (!$found) {
+        throw ("No suitable Visual Studio installation found")
+    }
+
     Write-Host "Executing: $VS_INIT_CMD $VS_INIT_ARGS"
 
     # https://github.com/Microsoft/vswhere/wiki/Start-Developer-Command-Prompt
@@ -85,66 +113,44 @@ if (!$env:NUMPY_VERSION) {
     throw "NUMPY_VERSION env var missing"
 }
 
-Init-VS
+Initialize-VS
+Initialize-Python
 
 Get-ChildItem env:
 
-$env:CONDA_ROOT = $pwd.Path + "\external\miniconda_$env:PYTHON_ARCH"
-& .\.github\scripts\install-miniconda.ps1
-
-& $env:CONDA_ROOT\shell\condabin\conda-hook.ps1
-
-exec { conda update --yes -n base -c defaults conda }
 
-exec { conda create --yes --name pyenv_build python=$env:PYTHON_VERSION numpy=$env:NUMPY_VERSION cython --force }
-exec { conda activate pyenv_build }
-
-# Check that we have the expected version and architecture for Python
-exec { python --version }
-exec { python -c "import struct; assert struct.calcsize('P') * 8 == $env:PYTHON_ARCH" }
-exec { python -c "import sys; print(sys.prefix)" }
-
-# output what's installed
-exec { python -m pip freeze }
-
-# Build the compiled extension.
-# -u disables output buffering which caused intermixing of lines
-# when the external tools were started  
+# Build the wheel.
+Create-And-Enter-VEnv build
+exec { python -m pip install --upgrade pip wheel setuptools }
+exec { python -m pip install --only-binary :all: numpy==$env:NUMPY_VERSION cython }
 exec { python -u setup.py bdist_wheel }
+Exit-VEnv
 
-# Necessary to avoid bug when switching to test env.
-exec { conda deactivate }
-
-# Import test on a minimal environment
-# (to catch DLL issues)
-exec { conda create --yes --name pyenv_minimal python=$env:PYTHON_VERSION --force }
-exec { conda activate pyenv_minimal }
-
-# Avoid using in-source package
-New-Item -Force -ItemType directory tmp_for_test | out-null
-cd tmp_for_test
-
+# Install and import in an empty environment.
+# This is to catch DLL issues that may be hidden with dependencies.
+Create-And-Enter-VEnv import-test
 python -m pip uninstall -y lensfunpy
-ls ..\dist\*.whl | % { exec { python -m pip install $_ } }
-exec { python -c "import lensfunpy" }
+ls dist\*.whl | % { exec { python -m pip install $_ } }
 
-# Necessary to avoid bug when switching to test env.
-exec { conda deactivate }
+# Avoid using in-source package during tests
+mkdir -f tmp_for_test | out-null
+pushd tmp_for_test
+exec { python -c "import lensfunpy" }
+popd
 
-# Test
-exec { conda create --yes --name pyenv_test python=$env:PYTHON_VERSION numpy scipy --force }
-exec { conda activate pyenv_test }
+Exit-VEnv
 
-# Check that we have the expected version and architecture for Python
-exec { python --version }
-exec { python -c "import struct; assert struct.calcsize('P') * 8 == $env:PYTHON_ARCH" }
-exec { python -c "import sys; print(sys.prefix)" }
+# Run test suite with all required and optional dependencies
+Create-And-Enter-VEnv testsuite
+exec { python -m pip install --only-binary :all: numpy scipy }
+python -m pip uninstall -y lensfunpy
+ls dist\*.whl | % { exec { python -m pip install $_ } }
+exec { python -m pip install -r dev-requirements.txt }
 
-# output what's installed
-exec { python -m pip freeze }
+# Avoid using in-source package during tests
+mkdir -f tmp_for_test | out-null
+pushd tmp_for_test
+exec { pytest --verbosity=3 -s ../test }
+popd
 
-python -m pip uninstall -y lensfunpy
-ls ..\dist\*.whl | % { exec { python -m pip install $_ } }
-exec { python -m pip install -r ..\dev-requirements.txt }
-exec { nosetests --verbosity=3 --nocapture ../test }
-cd ..
+Exit-VEnv

.github/scripts/check_sha256.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+path=$1
+expected=$2
+
+if [ "$(uname)" == "Darwin" ]; then
+    actual=$(shasum -a 256 "$path" | cut -d ' ' -f 1)
+elif [ "$(uname)" == "Linux" ]; then
+    actual=$(sha256sum "$path" | cut -d ' ' -f 1)
+else
+    echo "Unknown system: $(uname)"
+    exit 1
+fi
+
+if [ "$expected" != "$actual" ]; then 
+  echo "CHECKSUM MISMATCH: $path"
+  echo "Expected: $expected"
+  echo "Actual: $actual"
+  exit 1
+fi