Migrate all packages to openssl3 and python 3.11

.github/actions/build.sh

@@ -59,7 +59,7 @@ if [ -n "$API_KEY" ] && [ "$PUBLISH" == "true" ]; then
 fi
 
 # Build
-PACKAGES_TO_KEEP="ffmpeg4 ffmpeg5 ffmpeg6"
+PACKAGES_TO_KEEP="ffmpeg4 ffmpeg5 ffmpeg6 python310 python311"
 for package in ${build_packages}
 do
     echo "::group:: ---- build ${package}"

.github/actions/download.sh

@@ -6,9 +6,11 @@
 #
 # Functions:
 # - Download all referenced native and cross source files for packages to build.
+# - Download all referenced python wheels needed to build.
 
 set -o pipefail
 
+# Download regular cross/* sources
 if [ -z "${DOWNLOAD_PACKAGES}" ]; then
     echo "===> No packages to download. <==="
 else
@@ -19,3 +21,19 @@ else
         make -C ${download} download
     done
 fi
+
+echo ""
+
+# Download python wheel sources files
+build_packages="${NOARCH_PACKAGES} ${ARCH_PACKAGES}"
+
+if [ -z "${build_packages}" ]; then
+    echo "===> No wheels to download. <==="
+else
+    for package in ${build_packages}; do
+        echo "===> Download wheels: ${package}"
+        make -C spk/${package} wheeldownload
+    done
+fi
+
+echo ""

.github/actions/prepare.sh

@@ -6,7 +6,7 @@
 #
 # Functions:
 # - Evaluate all packages to build depending on files defined in ${GH_FILES}.
-# - ffmpeg (spk/ffmpeg4), ffmpeg5 and ffmpeg6 are moved to head of packages to built first if triggered by its own or a dependent.
+# - python310, python311, ffmpeg (spk/ffmpeg4), ffmpeg5 and ffmpeg6 are moved to head of packages to build first if triggered by its own or a dependent.
 # - Referenced native and cross packages of the packages to build are added to the download list.
 
 set -o pipefail
@@ -56,7 +56,7 @@ fi
 # remove duplicate packages
 packages=$(printf %s "${SPK_TO_BUILD}" | tr ' ' '\n' | sort -u | tr '\n' ' ')
 
-# for ffmpeg v4-6 find all packages that depends on them
+# for ffmpeg v4-6 find all packages that depend on them
 for i in {4..6}; do
     ffmpeg_dependent_packages=$(find spk/ -maxdepth 2 -mindepth 2 -name "Makefile" -exec grep -Ho "FFMPEG_VERSION = ${i}" {} \; | grep -Po ".*spk/\K[^/]*" | sort | tr '\n' ' ')
 
@@ -72,6 +72,22 @@ for i in {4..6}; do
     done
 done
 
+# for python (310, 311) find all packages that depend on them
+for py in python310 python311; do
+    python_dependent_packages=$(find spk/ -maxdepth 2 -mindepth 2 -name "Makefile" -exec grep -Ho "PYTHON_PACKAGE = ${py}" {} \; | grep -Po ".*spk/\K[^/]*" | sort | tr '\n' ' ')
+
+    # If packages contain a package that depends on python (or is python), then ensure
+    # relevant python spk is first in list
+    for package in ${packages}
+    do
+        if [ "$(echo ${py} ${python_dependent_packages} | grep -ow ${package})" != "" ]; then
+            packages_without_python=$(echo "${packages}" | tr ' ' '\n' | grep -v "${py}" | tr '\n' ' ')
+            packages="${py} ${packages_without_python}"
+            break
+        fi
+    done
+done
+
 # find all noarch packages
 all_noarch=$(find spk/ -maxdepth 2 -mindepth 2 -name "Makefile" -exec grep -Ho "override ARCH" {} \; | grep -Po ".*spk/\K[^/]*" | sort | tr '\n' ' ')
 

.github/workflows/build.yml

@@ -71,6 +71,8 @@ jobs:
         run: ./.github/actions/download.sh
         env:
           DOWNLOAD_PACKAGES: ${{ steps.dependencies.outputs.download_packages }}
+          ARCH_PACKAGES: ${{ needs.prepare.outputs.arch_packages }}
+          NOARCH_PACKAGES: ${{ needs.prepare.outputs.noarch_packages }}
 
   build:
     name: Build

cross/bazarr/Makefile

@@ -1,5 +1,5 @@
 PKG_NAME = bazarr
-PKG_VERS = 1.2.1
+PKG_VERS = 1.2.4
 PKG_EXT = zip
 PKG_DIST_NAME = $(PKG_NAME).$(PKG_EXT)
 PKG_DIST_FILE = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)

cross/bazarr/digests

@@ -1,3 +1,3 @@
-bazarr-1.2.1.zip SHA1 64e31750708a21520e2de1e7219e74e2db75c85b
-bazarr-1.2.1.zip SHA256 3ee54ad63acd8f16a011262f82a4417f1cec57f2b11614dd3b29623bcb26c1e7
-bazarr-1.2.1.zip MD5 fa57920c4eee9684b7a0f3a3847f4bd0
+bazarr-1.2.4.zip SHA1 e454c27c2ccf53bd7e59727da6382febc61dcd9a
+bazarr-1.2.4.zip SHA256 4dd05acde63fc3dbd211bb3f3a046d7688bfae20146a2d73ae633f03c79c6960
+bazarr-1.2.4.zip MD5 c1f4996968a79e9a0366fe1d7065a09a

cross/cryptography/Makefile

@@ -1,5 +1,5 @@
 PKG_NAME = cryptography
-PKG_VERS = 41.0.1
+PKG_VERS = 41.0.3
 PKG_EXT = tar.gz
 PKG_DIST_NAME = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
 PKG_DIST_SITE = https://files.pythonhosted.org/packages/source/c/$(PKG_NAME)
@@ -18,13 +18,4 @@ LICENSE  = BSD/ASL2
 
 WHEELS_BUILD_ARGS = --py-limited-api=$(PYTHON_LIMITED_API)
 
-# Mandatory PYO3_* variables for rust cross-compiling
-ENV += PYO3_CROSS_LIB_DIR=$(STAGING_INSTALL_PREFIX)/lib/
-ENV += PYO3_CROSS_INCLUDE_DIR=$(STAGING_INSTALL_PREFIX)/include/
-# Mandatory of using OPENSSL_*_DIR starting with
-# cryptography version >= 40
-# https://docs.rs/openssl/latest/openssl/#automatic
-ENV += OPENSSL_LIB_DIR=$(STAGING_INSTALL_PREFIX)/lib/
-ENV += OPENSSL_INCLUDE_DIR=$(STAGING_INSTALL_PREFIX)/include/
-
 include ../../mk/spksrc.python-wheel.mk

cross/cryptography/digests

@@ -1,3 +1,3 @@
-cryptography-41.0.1.tar.gz SHA1 25aba9235d2b4228417783ef1e56ab0293dd2711
-cryptography-41.0.1.tar.gz SHA256 d34579085401d3f49762d2f7d6634d6b6c2ae1242202e860f4d26b046e3a1006
-cryptography-41.0.1.tar.gz MD5 49d86f49c47e7b693700887efc117adc
+cryptography-41.0.3.tar.gz SHA1 80ad50220559469dc0cd43a7490d526028aecc54
+cryptography-41.0.3.tar.gz SHA256 6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34
+cryptography-41.0.3.tar.gz MD5 fbf930acd8de95780604c40c4e817a74

cross/cryptography_38/Makefile

@@ -0,0 +1,24 @@
+PKG_NAME = cryptography
+PKG_VERS = 38.0.3
+PKG_EXT = tar.gz
+PKG_DIST_NAME = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
+PKG_DIST_SITE = https://files.pythonhosted.org/packages/source/c/cryptography
+PKG_DIR = $(PKG_NAME)-$(PKG_VERS)
+
+DEPENDS = cross/openssl3 cross/cffi
+
+HOMEPAGE = https://github.com/pyca/cryptography
+COMMENT  = Provide cryptographic recipes and primitives to Python developers.
+LICENSE  = BSD/ASL2
+
+###
+### This cross/cryptography_38 is needed to generate python-limited abi3 wheels
+### Ref: https://github.com/pyca/cryptography/issues/7671
+### 
+### And it is required to link to cross/openssl3 since
+### cryptography < 40 is statically linked to openssl
+### and prebuilt wheels are linked to openssl 1.1
+
+WHEELS_BUILD_ARGS = --py-limited-api=$(PYTHON_LIMITED_API)
+
+include ../../mk/spksrc.python-wheel.mk

cross/cryptography_38/digests

@@ -0,0 +1,3 @@
+cryptography-38.0.3.tar.gz SHA1 fa8da1574efb4a44158825a8e8c60e050142e789
+cryptography-38.0.3.tar.gz SHA256 bfbe6ee19615b07a98b1d2287d6a6073f734735b49ee45b11324d85efc4d5cbd
+cryptography-38.0.3.tar.gz MD5 2148f1283f22df0677e204e46bccaf06

cross/duply/Makefile

@@ -1,5 +1,5 @@
 PKG_NAME = duply
-PKG_VERS = 2.4.1
+PKG_VERS = 2.4.3
 PKG_SHORT_VERS = $(word 1,$(subst ., ,$(PKG_VERS))).$(word 2,$(subst ., ,$(PKG_VERS)))
 PKG_EXT = tgz
 PKG_DIST_NAME = $(PKG_NAME)_$(PKG_VERS).$(PKG_EXT)

cross/duply/digests

@@ -1,3 +1,3 @@
-duply_2.4.1.tgz SHA1 98790dafb92f8441045ce430f4ad9ed80bd895f6
-duply_2.4.1.tgz SHA256 03fd28b06206505edf0e0820d098208708842999f21b1d132461e34e2c0b6973
-duply_2.4.1.tgz MD5 7d579e5a922c21b59e059a2dd4d2383b
+duply_2.4.3.tgz SHA1 278e45e0c033a2f24adfe868339114c7365ea318
+duply_2.4.3.tgz SHA256 867388531ddda2a5cdb0a2af77bde03261491c03a406b60fe26b96b4d92627a3
+duply_2.4.3.tgz MD5 335f65f0af7610fb248f7f549ec81cab

cross/libtorrent-rakshasa/Makefile

@@ -6,7 +6,7 @@ PKG_DIST_SITE = https://github.com/rakshasa/libtorrent/archive
 PKG_DIST_FILE = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
 PKG_DIR = $(PKG_NAME)-$(PKG_VERS)
 
-DEPENDS = cross/cppunit cross/openssl cross/libsigc++ cross/zlib
+DEPENDS = cross/cppunit cross/openssl3 cross/libsigc++ cross/zlib
 
 HOMEPAGE = https://rakshasa.github.io
 COMMENT  = LibTorrent is a BitTorrent library written in C++ for *nix, with a focus on high performance and good code.

cross/nmap/Makefile

@@ -5,7 +5,7 @@ PKG_DIST_NAME = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
 PKG_DIST_SITE = https://nmap.org/dist
 PKG_DIR = $(PKG_NAME)-$(PKG_VERS)
 
-DEPENDS = cross/libpcap cross/openssl
+DEPENDS = cross/libpcap cross/openssl3
 
 # latest version for OLD_PPC_ARCHS is 7.92
 UNSUPPORTED_ARCHS = $(OLD_PPC_ARCHS)
@@ -15,7 +15,12 @@ COMMENT  = Nmap ("Network Mapper") is a free and open source utility for network
 LICENSE  = https://svn.nmap.org/nmap/COPYING
 
 GNU_CONFIGURE = 1
-CONFIGURE_ARGS = --without-zenmap --with-liblua=included --with-libpcre=included
+CONFIGURE_ARGS  = --without-zenmap
+CONFIGURE_ARGS += --with-liblua=included
+CONFIGURE_ARGS += --with-libpcre=included
+ifneq ($(wildcard $(OPENSSL_STAGING_PREFIX)),)
+CONFIGURE_ARGS += --with-openssl=$(OPENSSL_STAGING_PREFIX)
+endif
 
 ADDITIONAL_CPPFLAGS = -O3
 

cross/python310/Makefile

@@ -53,9 +53,7 @@ CONFIGURE_ARGS += ac_cv_file__dev_ptmx=no
 CONFIGURE_ARGS += ac_cv_file__dev_ptc=no
 CONFIGURE_ARGS += ac_cv_have_long_long_format=yes
 
-# keep both openssl until all python packages
-# have been republished using openssl3
-DEPENDS += cross/openssl cross/openssl3
+DEPENDS += cross/openssl3
 CONFIGURE_ARGS += --with-ssl-default-suites=openssl
 
 DEPENDS += cross/gdbm
@@ -138,10 +136,12 @@ python310_install:
 CROSSENV_WHEELS  = setuptools-rust==1.6.0
 CROSSENV_WHEELS += setuptools-scm==7.1.0
 CROSSENV_WHEELS += cffi==1.15.1
-CROSSENV_WHEELS += poetry==1.5.1
 CROSSENV_WHEELS += Cython==0.29.35
 CROSSENV_WHEELS += flit==3.9.0
-CROSSENV_WHEELS += cryptography==41.0.1
+ifneq ($(findstring $(ARCH),$(OLD_PPC_ARCHS)),$(ARCH))
+CROSSENV_WHEELS += cryptography==41.0.3
+CROSSENV_WHEELS += poetry==1.5.1
+endif
 ifneq ($(findstring $(ARCH),$(ARMv5_ARCHS) $(OLD_PPC_ARCHS)),$(ARCH))
 CROSSENV_WHEELS += maturin==1.1.0
 endif
@@ -152,7 +152,7 @@ endif
 python310_post_install: $(WORK_DIR)/python-cc.mk
 	mkdir -p $(PYTHON_LIB_CROSS)
 	cp -R $(HOSTPYTHON_LIB_NATIVE) $(PYTHON_LIB_CROSS)/../
-	@$(RUN) $(PYTHON_NATIVE) -m crossenv $(STAGING_INSTALL_PREFIX)/bin/python$(PKG_VERS_MAJOR_MINOR) --cc $${CC} --cxx $${CXX} --ar $${AR} --sysroot $${SYSROOT} --env LIBRARY_PATH= --manylinux manylinux2014 $(WORK_DIR)/crossenv/
+	@$(RUN) $(PYTHON_NATIVE) -m crossenv $(STAGING_INSTALL_PREFIX)/bin/python$(PKG_VERS_MAJOR_MINOR) --cc $(TC_PATH)$(TC_PREFIX)gcc --cxx $(TC_PATH)$(TC_PREFIX)c++ --ar $(TC_PATH)$(TC_PREFIX)ar --sysroot $(TC_SYSROOT) --env LIBRARY_PATH= --manylinux manylinux2014 $(WORK_DIR)/crossenv/
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) wget https://bootstrap.pypa.io/get-pip.py
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-python get-pip.py "pip==23.1.2" --no-setuptools --no-wheel --disable-pip-version-check
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) python get-pip.py "pip==23.1.2" --no-setuptools --no-wheel --disable-pip-version-check

cross/python311/Makefile

@@ -6,6 +6,10 @@ PKG_DIST_NAME = Python-$(PKG_VERS).$(PKG_EXT)
 PKG_DIST_SITE = https://www.python.org/ftp/python/$(PKG_VERS)
 PKG_DIR = Python-$(PKG_VERS)
 
+# Platform "powerpc-none-linux-gnuspe" with compiler "gcc" is not supported by the
+# CPython core team, see https://peps.python.org/pep-0011/ for more information.
+UNSUPPORTED_ARCHS = $(OLD_PPC_ARCHS)
+
 BUILD_DEPENDS = native/$(PKG_NAME)
 
 DEPENDS  = cross/zlib cross/sqlite cross/readline cross/ncursesw cross/bzip2 cross/xz
@@ -34,29 +38,19 @@ include ../../mk/spksrc.archs.mk
 # optionally generate optimized code
 ifeq ($(strip $(PYTHON_OPTIMIZE)),1)
 CONFIGURE_ARGS += --enable-optimizations
-# old compilers fail with unrecognized command line options: -flto -fuse-linker-plugin -ffat-lto-objects -flto-partition=none
-ifneq ($(findstring $(ARCH),$(OLD_PPC_ARCHS)),$(ARCH))
 # Enable Link-Time Optimization
 CONFIGURE_ARGS += --with-lto
-endif
 else
 # test modules are required for optimization only
 CONFIGURE_ARGS += --disable-test-modules
 endif
 
-ifeq ($(findstring $(ARCH),$(OLD_PPC_ARCHS)),$(ARCH))
-# older gcc does not know -Wno-unused-result
-CONFIGURE_ARGS += ac_cv_disable_unused_result_warning=no
-endif
-
 CONFIGURE_ARGS += ac_cv_buggy_getaddrinfo=no
 CONFIGURE_ARGS += ac_cv_file__dev_ptmx=no
 CONFIGURE_ARGS += ac_cv_file__dev_ptc=no
 CONFIGURE_ARGS += ac_cv_have_long_long_format=yes
 
-# keep both openssl until all python packages
-# have been republished using openssl3
-DEPENDS += cross/openssl cross/openssl3
+DEPENDS += cross/openssl3
 CONFIGURE_ARGS += --with-ssl-default-suites=openssl
 
 DEPENDS += cross/gdbm
@@ -134,15 +128,30 @@ python311_install:
 	@install -m 644 src/mime.types $(STAGING_INSTALL_PREFIX)/etc/
 	$(RUN) _PYTHON_HOST_PLATFORM=$(TC_TARGET) $(MAKE) install prefix=$(STAGING_INSTALL_PREFIX)
 
-# wheels to install in crossenv
-CROSSENV_WHEELS  = setuptools-rust==1.6.0
-CROSSENV_WHEELS += setuptools-scm==7.1.0
-CROSSENV_WHEELS += cffi==1.15.1
-CROSSENV_WHEELS += poetry==1.5.1
+# default wheels to install in crossenv
+CROSSENV_WHEELS  = cffi==1.15.1
+CROSSENV_WHEELS += cryptography==41.0.3
 CROSSENV_WHEELS += Cython==0.29.35
 CROSSENV_WHEELS += flit==3.9.0
-CROSSENV_WHEELS += cryptography==41.0.1
-ifneq ($(findstring $(ARCH),$(ARMv5_ARCHS) $(OLD_PPC_ARCHS)),$(ARCH))
+CROSSENV_WHEELS += poetry==1.5.1
+CROSSENV_WHEELS += scikit-build==0.17.6
+CROSSENV_WHEELS += setuptools-rust==1.6.0
+CROSSENV_WHEELS += setuptools-scm==7.1.0
+
+# [numpy] <= 1.21.6 (armv5)  - gcc-4.6.4 - unsupported
+# [numpy] <= 1.22.4 (armv7l) - gcc-4.8.3 - unsupported
+ifneq ($(findstring $(ARCH),$(ARMv5_ARCHS) $(ARMv7L_ARCHS)),$(ARCH))
+# [numpy] <= 1.24.4 last working version with gcc-4.9
+ifeq ($(call version_le, $(TC_GCC), 5.0),1)
+CROSSENV_WHEELS += numpy==1.24.4
+# [numpy] >= 1.25.0 requires c++17
+else ifeq ($(call version_gt, $(TC_GCC), 5.0),1)
+CROSSENV_WHEELS += numpy==1.25.1
+endif
+endif
+
+# [maturin]
+ifneq ($(findstring $(ARCH),$(ARMv5_ARCHS)),$(ARCH))
 CROSSENV_WHEELS += maturin==1.1.0
 endif
 
@@ -152,7 +161,7 @@ endif
 python311_post_install: $(WORK_DIR)/python-cc.mk
 	mkdir -p $(PYTHON_LIB_CROSS)
 	cp -R $(HOSTPYTHON_LIB_NATIVE) $(PYTHON_LIB_CROSS)/../
-	@$(RUN) $(PYTHON_NATIVE) -m crossenv $(STAGING_INSTALL_PREFIX)/bin/python$(PKG_VERS_MAJOR_MINOR) --cc $${CC} --cxx $${CXX} --ar $${AR} --sysroot $${SYSROOT} --env LIBRARY_PATH= --manylinux manylinux2014 $(WORK_DIR)/crossenv/
+	@$(RUN) $(PYTHON_NATIVE) -m crossenv $(STAGING_INSTALL_PREFIX)/bin/python$(PKG_VERS_MAJOR_MINOR) --cc $(TC_PATH)$(TC_PREFIX)gcc --cxx $(TC_PATH)$(TC_PREFIX)c++ --ar $(TC_PATH)$(TC_PREFIX)ar --sysroot $(TC_SYSROOT) --env LIBRARY_PATH= --manylinux manylinux2014 $(WORK_DIR)/crossenv/
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) wget https://bootstrap.pypa.io/get-pip.py
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) build-python get-pip.py "pip==23.1.2" --no-setuptools --no-wheel --disable-pip-version-check
 	. $(WORK_DIR)/crossenv/bin/activate && $(RUN) python get-pip.py "pip==23.1.2" --no-setuptools --no-wheel --disable-pip-version-check

cross/uamqp/Makefile

@@ -0,0 +1,22 @@
+PKG_NAME = uamqp
+PKG_VERS = 1.6.0
+PKG_EXT = tar.gz
+PKG_DIST_NAME = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
+PKG_DIST_SITE = https://files.pythonhosted.org/packages/source/u/uamqp
+PKG_DIR = $(PKG_NAME)-$(PKG_VERS)
+
+DEPENDS = cross/openssl3 cross/cffi
+
+HOMEPAGE = https://github.com/Azure/azure-uamqp-python
+COMMENT  = AMQP 1.0 client library for Python.
+LICENSE  = MIT
+
+POST_PATCH_TARGET = uamqp_post_patch
+
+include ../../mk/spksrc.python-wheel.mk
+
+.PHONY: uamqp_post_patch
+# https://github.com/Azure/azure-uamqp-python/issues/276#issuecomment-924890556
+uamqp_post_patch:
+	@$(MSG) "Patch CMakeLists.txt files to avoid (deprecation) warnings as errors."
+	@$(RUN) ; grep -rl "\-Werror" . | xargs sed -i 's/-Werror//'

cross/uamqp/digests

@@ -0,0 +1,3 @@
+uamqp-1.6.0.tar.gz SHA1 3f1cbb519c2252a73a92b7d89f179a6cd43d9469
+uamqp-1.6.0.tar.gz SHA256 2c31b74a10856eccf258d73c4d08e5cac4d6060a34b9834890bfd42bcb138350
+uamqp-1.6.0.tar.gz MD5 e052ebabbc9d72099c33fba6d9f66ec2