Skip uninstallable tags in QPY backwards compatibility tests (backport …

…#13202) (#13217)

* Skip uninstallable tags in QPY backwards compatibility tests (#13202)

* Skip uninstallable tags in QPY backwards compatibility tests

When a tag has been made, but the package has not yet landed on PyPI,
the QPY job fails in the environment-building step.  This is not
actually a failure of the QPY backwards-compatibility guarantees, and
it isn't the job of the QPY tests to detect a bad tag anyway.

* Query PyPI to find versions to test

The logic we actually want for versions to test is "find the versions
that _should_ be installable", rather than "try it and see"; the latter
is susceptible to silently suppressing errors.  This new form now
queries PyPI to find what versions of Qiskit are available in binary
distributions for this platform, and filters based on that.

* Include release candidates in testing

(cherry picked from commit 1344cdd)

# Conflicts:
#	test/qpy_compat/process_version.sh
#	test/qpy_compat/run_tests.sh

* Resolve conflicts

* Add qpy_teest_constraints.txt

---------

Co-authored-by: Jake Lishman <[email protected]>
Co-authored-by: Eli Arbel <[email protected]>

.azure/lint_docs_qpy-linux.yml

@@ -61,6 +61,6 @@ jobs:
           pushd test/qpy_compat
           ./run_tests.sh
           popd
-          mkdir qpy_files || :
+          mkdir -p qpy_files
           mv test/qpy_compat/qpy_* qpy_files/.
         displayName: 'Run QPY backwards compat tests'

test/qpy_compat/get_versions.py

@@ -0,0 +1,99 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2024.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""List the versions of Qiskit and Terra that should be tested for QPY compatibility."""
+
+import json
+import sys
+import urllib.request
+
+import packaging.version
+import packaging.tags
+
+import qiskit
+
+
+def tags_from_wheel_name(wheel: str):
+    """Extract the wheel tag from its filename."""
+    assert wheel.lower().endswith(".whl")
+    # For more information, see:
+    # - https://packaging.python.org/en/latest/specifications/binary-distribution-format/
+    # - https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/
+    #
+    # In particular, take note that a wheel's filename can include "compressed tag sets" (and our
+    # Linux wheels generally do this), which is why this function returns an iterable of tags.
+    _prefix, interpreters, abis, platforms = wheel[:-4].rsplit("-", 3)
+    yield from (
+        packaging.tags.Tag(interpreter, abi, platform)
+        for interpreter in interpreters.split(".")
+        for abi in abis.split(".")
+        for platform in platforms.split(".")
+    )
+
+
+def available_versions():
+    """Get all the versions of Qiskit that support exporting QPY, and are installable with the
+    active version of Python on this platform."""
+    our_version = packaging.version.parse(qiskit.__version__)
+    supported_tags = set(packaging.tags.sys_tags())
+
+    def available_versions_for_package(package, min_version=None, max_version=None):
+        with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/json") as fd:
+            data = json.load(fd)
+        min_version = min_version and packaging.version.parse(min_version)
+        max_version = max_version and packaging.version.parse(max_version)
+        for other_version, payload in data["releases"].items():
+            other_version = packaging.version.parse(other_version)
+            if min_version is not None and other_version < min_version:
+                continue
+            if max_version is not None and other_version >= max_version:
+                continue
+            if other_version > our_version:
+                continue
+            if other_version.pre is not None and other_version.pre[0] in ("a", "b"):
+                # We skip alpha and beta prereleases, but we currently want to test for
+                # compatibility with release candidates.
+                continue
+            # Note: this ignores versions that are uninstallable because we're using a Python
+            # version that's too new, which can be a problem for the oldest Terras, especially from
+            # before we built for abi3.  We're not counting sdists, since if we didn't release a
+            # compatible wheel for the current Python version, there's no guarantee it'll install.
+            if not any(
+                tag in supported_tags
+                for release in payload
+                if release["packagetype"] == "bdist_wheel"
+                for tag in tags_from_wheel_name(release["filename"])
+            ):
+                print(
+                    f"skipping '{other_version}', which has no installable binary artifacts",
+                    file=sys.stderr,
+                )
+                continue
+            yield other_version
+
+    yield from (
+        ("qiskit-terra", version)
+        for version in available_versions_for_package("qiskit-terra", "0.18.0", "1.0.0")
+    )
+    yield from (
+        ("qiskit", version) for version in available_versions_for_package("qiskit", "1.0.0")
+    )
+
+
+def main():
+    """main"""
+    for package, version in available_versions():
+        print(package, version)
+
+
+if __name__ == "__main__":
+    main()

test/qpy_compat/process_version.sh

@@ -14,46 +14,50 @@
 set -e
 set -x
 
-# version is the source version, this is the release with which to generate
-# qpy files with to load with the version under test
-version=$1
-parts=( ${version//./ } )
-# qiskit_version is the version under test, We're testing that we can correctly
-# read the qpy files generated with source version with this version.
-qiskit_version=`./qiskit_venv/bin/python -c "import qiskit;print(qiskit.__version__)"`
-qiskit_parts=( ${qiskit_version//./ } )
+function usage {
+    echo "usage: ${BASH_SOURCE[0]} -p /path/to/qiskit/python <package> <version>" 1>&2
+    exit 1
+}
 
-
-# If source version is less than 0.18 QPY didn't exist yet so exit fast
-if [[ ${parts[0]} -eq 0 && ${parts[1]} -lt 18 ]] ; then
-    exit 0
+python="python"
+while getopts "p:" opt; do
+    case "$opt" in
+        p)
+            python="$OPTARG"
+            ;;
+        *)
+            usage
+            ;;
+    esac
+done
+shift "$((OPTIND-1))"
+if [[ $# != 2 ]]; then
+    usage
 fi
 
-# Exclude any non-rc pre-releases as they don't have stable API guarantees
-if [[ $version == *"b"* || $version == *"a"* ]] ; then
-    exit 0
-fi
+# `package` is the name of the Python distribution to install (qiskit or qiskit-terra). `version` is
+# the source version: the release with which to generate qpy files with to load with the version
+# under test.
+package="$1"
+version="$2"
 
-# If the source version is newer than the version under test exit fast because
-# there is no QPY compatibility for loading a qpy file generated from a newer
-# release with an older release of Qiskit.
-if ! ./qiskit_venv/bin/python ./compare_versions.py "$version" "$qiskit_version" ; then
-    exit 0
-fi
+our_dir="$(realpath -- "$(dirname -- "${BASH_SOURCE[0]}")")"
+cache_dir="$(pwd -P)/qpy_$version"
+venv_dir="$(pwd -P)/${version}"
 
-if [[ ! -d qpy_$version ]] ; then
-    echo "Building venv for qiskit-terra $version"
-    python -m venv $version
-    ./$version/bin/pip install "qiskit-terra==$version"
-    mkdir qpy_$version
-    pushd qpy_$version
-    echo "Generating qpy files with qiskit-terra $version"
-    ../$version/bin/python ../test_qpy.py generate --version=$version
+if [[ ! -d $cache_dir ]] ; then
+    echo "Building venv for $package==$version"
+    "$python" -m venv "$venv_dir"
+    "$venv_dir/bin/pip" install -c "${our_dir}/qpy_test_constraints.txt" "${package}==${version}"
+    mkdir "$cache_dir"
+    pushd "$cache_dir"
+    echo "Generating QPY files with $package==$version"
+    "$venv_dir/bin/python" "${our_dir}/test_qpy.py" generate --version="$version"
 else
     echo "Using cached QPY files for $version"
-    pushd qpy_$version
+    pushd "${cache_dir}"
 fi
-echo "Loading qpy files from $version with dev qiskit-terra"
-../qiskit_venv/bin/python ../test_qpy.py load --version=$version
+echo "Loading qpy files from $version with dev Qiskit"
+"$python" "${our_dir}/test_qpy.py" load --version="$version"
 popd
-rm -rf ./$version
+rm -rf "$venv_dir"

test/qpy_compat/qpy_test_constraints.txt

@@ -0,0 +1,2 @@
+numpy===1.24.4
+scipy===1.10.1

test/qpy_compat/run_tests.sh

@@ -12,19 +12,25 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 set -e
+set -o pipefail
 set -x
 
 # Set fixed hash seed to ensure set orders are identical between saving and
 # loading.
 export PYTHONHASHSEED=$(python -S -c "import random; print(random.randint(1, 4294967295))")
 echo "PYTHONHASHSEED=$PYTHONHASHSEED"
 
-python -m venv qiskit_venv
-qiskit_venv/bin/pip install ../..
+our_dir="$(realpath -- "$(dirname -- "${BASH_SOURCE[0]}")")"
 
-parallel bash ./process_version.sh ::: `git tag --sort=-creatordate`
+qiskit_venv="$(pwd -P)/qiskit_venv"
+qiskit_python="$qiskit_venv/bin/python"
+python -m venv "$qiskit_venv"
+# `packaging` is needed for the `get_versions.py` script.
+"$qiskit_venv/bin/pip" install -c "$our_dir/../../constraints.txt" "$our_dir/../.." packaging
+
+"$qiskit_python" "$our_dir/get_versions.py" | parallel --colsep=" " bash "$our_dir/process_version.sh" -p "$qiskit_python"
 
 # Test dev compatibility
-dev_version=`qiskit_venv/bin/python -c 'import qiskit;print(qiskit.__version__)'`
-qiskit_venv/bin/python test_qpy.py generate --version=$dev_version
-qiskit_venv/bin/python test_qpy.py load --version=$dev_version
+dev_version="$("$qiskit_python" -c 'import qiskit; print(qiskit.__version__)')"
+"$qiskit_python" "$our_dir/test_qpy.py" generate --version="$dev_version"
+"$qiskit_python" "$our_dir/test_qpy.py" load --version="$dev_version"