add python3.12 as it is default one on Ubuntu 24.04 (#3349)

ROCm · Aug 13, 2024 · 03bd46c · 03bd46c
1 parent 7829432
commit 03bd46c
Show file tree

Hide file tree

Showing 12 changed files with 148 additions and 19 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -4,6 +4,7 @@
 # Allow files and directories
 !*.txt
 !*.ini
+!/tools/*.txt
 !/tools/*.sh
 !/test/onnx/.onnxrt-commit
 !/docs/sphinx/*.txt
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -40,8 +40,8 @@ jobs:
       - name: Create Image Tag
         id: image_hash
         run: |
-          echo "imagetag=hip-clang-${{hashFiles('**/hip-clang.docker', '**/*requirements.txt', '**/install_prereqs.sh', '**/rbuild.ini')}}" >> $GITHUB_OUTPUT
-          echo "imagetag_sles=hip-clang-${{hashFiles('**/tools/docker/sles.docker', '**/*requirements.txt', '**/install_prereqs.sh', '**/rbuild.ini')}}" >> $GITHUB_OUTPUT
+          echo "imagetag=hip-clang-${{hashFiles('**/hip-clang.docker', '**/*requirements.txt', '**/requirements-py.txt', '**/install_prereqs.sh', '**/rbuild.ini')}}" >> $GITHUB_OUTPUT
+          echo "imagetag_sles=hip-clang-${{hashFiles('**/tools/docker/sles.docker', '**/*requirements.txt','**/requirements-py.txt', '**/install_prereqs.sh', '**/rbuild.ini')}}" >> $GITHUB_OUTPUT
 
       - name: Check if image is built already
         id: check_image

diff --git a/Dockerfile b/Dockerfile
@@ -90,7 +90,8 @@ ENV ONNX_HOME=/.onnx
 RUN mkdir -p $ONNX_HOME/models && chmod 777 $ONNX_HOME/models
 
 COPY ./tools/install_prereqs.sh /
-RUN /install_prereqs.sh /usr/local / && rm /install_prereqs.sh
+COPY ./tools/requirements-py.txt /requirements-py.txt
+RUN /install_prereqs.sh /usr/local / && rm /install_prereqs.sh && rm /requirements-py.txt
 RUN test -f /usr/local/hash || exit 1
 
 # Install yapf

diff --git a/cmake/PythonModules.cmake b/cmake/PythonModules.cmake
@@ -1,7 +1,7 @@
 #####################################################################################
 # The MIT License (MIT)
 #
-# Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
+# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -86,7 +86,7 @@ function(py_add_module NAME)
     )
 
 endfunction()
-set(PYTHON_SEARCH_VERSIONS 3.5 3.6 3.7 3.8 3.9 3.10)
+set(PYTHON_SEARCH_VERSIONS 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12)
 set(PYTHON_DISABLE_VERSIONS "" CACHE STRING "")
 foreach(PYTHON_DISABLE_VERSION ${PYTHON_DISABLE_VERSIONS})
     list(REMOVE_ITEM PYTHON_SEARCH_VERSIONS ${PYTHON_DISABLE_VERSION})

diff --git a/hip-clang.docker b/hip-clang.docker
@@ -60,5 +60,6 @@ ADD requirements.txt /requirements.txt
 ADD rbuild.ini /rbuild.ini
 
 COPY ./tools/install_prereqs.sh /
-RUN /install_prereqs.sh /usr/local / && rm /install_prereqs.sh
+COPY ./tools/requirements-py.txt /
+RUN /install_prereqs.sh /usr/local / && rm /install_prereqs.sh && rm /requirements-py.txt
 
diff --git a/test/py/onnx_backend_test.py b/test/py/onnx_backend_test.py
@@ -42,7 +42,12 @@ def __init__(self, backend, parent_module=None):
         super(MIGraphXBackendTest, self).__init__(backend, parent_module)
 
     @classmethod
-    def assert_similar_outputs(cls, ref_outputs, outputs, rtol, atol):
+    def assert_similar_outputs(cls,
+                               ref_outputs,
+                               outputs,
+                               rtol,
+                               atol,
+                               model_dir=None):
         prog_string = c2.get_program()
         np.testing.assert_equal(len(ref_outputs),
                                 len(outputs),
@@ -556,9 +561,92 @@ def disabled_tests_onnx_1_14_0(backend_test):
     backend_test.exclude(r'test_wrap_pad_cpu')
 
 
+def disabled_tests_onnx_1_16_0(backend_test):
+    backend_test.exclude(r'test_dequantizelinear_blocked_cpu')
+    backend_test.exclude(r'test_dft_axis_opset19_cpu')
+    backend_test.exclude(r'test_dft_inverse_opset19_cpu')
+    backend_test.exclude(r'test_dft_opset19_cpu')
+    backend_test.exclude(
+        r'test_gridsample_bicubic_align_corners_0_additional_1_cpu')
+    backend_test.exclude(
+        r'test_gridsample_bicubic_align_corners_1_additional_1_cpu')
+    backend_test.exclude(
+        r'test_gridsample_volumetric_bilinear_align_corners_0_cpu')
+    backend_test.exclude(
+        r'test_gridsample_volumetric_bilinear_align_corners_1_cpu')
+    backend_test.exclude(
+        r'test_gridsample_volumetric_nearest_align_corners_0_cpu')
+    backend_test.exclude(
+        r'test_gridsample_volumetric_nearest_align_corners_1_cpu')
+    backend_test.exclude(r'test_group_normalization_epsilon_cpu')
+    backend_test.exclude(r'test_group_normalization_example_cpu')
+    backend_test.exclude(r'test_quantizelinear_int16_cpu')
+    backend_test.exclude(r'test_quantizelinear_uint16_cpu')
+    backend_test.exclude(r'test_qlinearmatmul_2D_int8_float16_cpu')
+    backend_test.exclude(r'test_qlinearmatmul_2D_int8_float32_cpu')
+    backend_test.exclude(r'test_qlinearmatmul_2D_uint8_float16_cpu')
+    backend_test.exclude(r'test_qlinearmatmul_2D_uint8_float32_cpu')
+    backend_test.exclude(r'test_qlinearmatmul_3D_int8_float16_cpu')
+    backend_test.exclude(r'test_qlinearmatmul_3D_int8_float32_cpu')
+    backend_test.exclude(r'test_qlinearmatmul_3D_uint8_float16_cpu')
+    backend_test.exclude(r'test_qlinearmatmul_3D_uint8_float32_cpu')
+    backend_test.exclude(r'test_quantizelinear_blocked_asymmetric_cpu')
+    backend_test.exclude(r'test_quantizelinear_blocked_symmetric_cpu')
+    backend_test.exclude(r'test_reduce_l1_empty_set_cpu')
+    backend_test.exclude(r'test_reduce_l1_empty_set_expanded_cpu')
+    backend_test.exclude(r'test_reduce_l2_empty_set_cpu')
+    backend_test.exclude(r'test_reduce_l2_empty_set_expanded_cpu')
+    backend_test.exclude(r'test_reduce_log_sum_empty_set_cpu')
+    backend_test.exclude(r'test_reduce_log_sum_empty_set_expanded_cpu')
+    backend_test.exclude(r'test_reduce_log_sum_exp_empty_set_cpu')
+    backend_test.exclude(r'test_reduce_log_sum_exp_empty_set_expanded_cpu')
+    backend_test.exclude(r'test_reduce_max_bool_inputs_cpu')
+    backend_test.exclude(r'test_reduce_min_bool_inputs_cpu')
+    # TODO: empty set ReduceOps tests are generating dynamic shapes
+    backend_test.exclude(r'test_reduce_min_empty_set_cpu')
+    backend_test.exclude(r'test_reduce_prod_empty_set_cpu')
+    backend_test.exclude(r'test_reduce_sum_empty_set_cpu')
+    backend_test.exclude(
+        r'test_reduce_sum_empty_set_non_reduced_axis_zero_cpu')
+    backend_test.exclude(r'test_reduce_sum_square_empty_set_cpu')
+    backend_test.exclude(r'test_reduce_sum_square_empty_set_expanded_cpu')
+    # TODO: Pooling tests are failing with shape mismatches, look into it.
+    backend_test.exclude(
+        r'test_averagepool_3d_dilations_large_count_include_pad_is_0_ceil_mode_is_True_cpu'
+    )
+    backend_test.exclude(
+        r'test_averagepool_3d_dilations_large_count_include_pad_is_1_ceil_mode_is_True_cpu'
+    )
+    backend_test.exclude(r'test_maxpool_2d_ceil_output_size_reduce_by_one_cpu')
+    backend_test.exclude(r'test_maxpool_3d_dilations_use_ref_impl_large_cpu')
+
+
+def disabled_tests_int4(backend_test):
+    # quantizelinear
+    backend_test.exclude(r'test_quantizelinear_int4')
+    backend_test.exclude(r'test_quantizelinear_uint4')
+    # dequantizelinear
+    backend_test.exclude(r'test_dequantizelinear_int4')
+    backend_test.exclude(r'test_dequantizelinear_uint4')
+    # cast
+    backend_test.exclude(r'test_cast_FLOAT16_to_UINT4')
+    backend_test.exclude(r'test_cast_FLOAT16_to_INT4')
+    backend_test.exclude(r'test_cast_FLOAT_to_INT4')
+    backend_test.exclude(r'test_cast_FLOAT_to_UINT4')
+    backend_test.exclude(r'test_cast_INT4_to_FLOAT')
+    backend_test.exclude(r'test_cast_INT4_to_FLOAT16')
+    backend_test.exclude(r'test_cast_INT4_to_INT8')
+    backend_test.exclude(r'test_cast_UINT4_to_FLOAT')
+    backend_test.exclude(r'test_cast_UINT4_to_FLOAT16')
+    backend_test.exclude(r'test_cast_UINT4_to_INT8')
+    backend_test.exclude(r'test_cast_UINT4_to_UINT8_cpu')
+
+
 def disabled_tests_float8(backend_test):
     # e4m3fn (Prototensor data type 17 not supported)
     backend_test.exclude(r'test_dequantizelinear_e4m3fn_cpu')
+    backend_test.exclude(r'test_dequantizelinear_e4m3fn_float16_cpu')
+    backend_test.exclude(r'test_dequantizelinear_e4m3fn_zero_point_cpu')
     backend_test.exclude(r'test_quantizelinear_e4m3fn_cpu')
     backend_test.exclude(r'test_cast_FLOAT16_to_FLOAT8E4M3FN_cpu')
     backend_test.exclude(r'test_cast_FLOAT8E4M3FN_to_FLOAT16_cpu')
@@ -1074,6 +1162,9 @@ def create_backend_test(testname=None, target_device=None):
         # Remove when float8 is supported
         disabled_tests_float8(backend_test)
 
+        # Remove Int4 tests
+        disabled_tests_int4(backend_test)
+
         # Remove when dynamic shapes are supported
         disabled_tests_dynamic_shape(backend_test)
 
@@ -1102,6 +1193,9 @@ def create_backend_test(testname=None, target_device=None):
         if version.parse(onnx.__version__) >= version.parse("1.14.0"):
             disabled_tests_onnx_1_14_0(backend_test)
 
+        if version.parse(onnx.__version__) >= version.parse("1.16.0"):
+            disabled_tests_onnx_1_16_0(backend_test)
+
 
 # import all test cases at global scope to make
 # them visible to python.unittest.

diff --git a/test/py/requirements-onnx.txt b/test/py/requirements-onnx.txt
@@ -21,9 +21,10 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #####################################################################################
-
-onnx==1.14.1
+onnx==1.16;python_version>="3.11"
+onnx==1.14.1;python_version<"3.11"
 protobuf==3.20.2
-numpy==1.21.6
+numpy==1.26.4;python_version>="3.11"
+numpy==1.21.6;python_version<"3.11"
 packaging==23.0
 pytest==6.0.1
diff --git a/tools/accuracy/requirements.txt b/tools/accuracy/requirements.txt
@@ -21,5 +21,6 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #####################################################################################
-numpy==1.21.6
+numpy==1.26.4;python_version>="3.11"
+numpy==1.21.6;python_version<"3.11"
 onnxruntime==1.17.3
diff --git a/tools/docker/sles.docker b/tools/docker/sles.docker
@@ -47,4 +47,5 @@ ADD requirements.txt /requirements.txt
 ADD rbuild.ini /rbuild.ini
 
 COPY ./tools/install_prereqs.sh /
-RUN /install_prereqs.sh /usr/local / && rm /install_prereqs.sh
+COPY ./tools/requirements-py.txt /
+RUN /install_prereqs.sh /usr/local / && rm /install_prereqs.sh && rm /requirements-py.txt
diff --git a/tools/docker/ubuntu_2204.dockerfile b/tools/docker/ubuntu_2204.dockerfile
@@ -80,7 +80,8 @@ ADD requirements.txt /requirements.txt
 ADD rbuild.ini /rbuild.ini
 
 COPY ./tools/install_prereqs.sh /
-RUN /install_prereqs.sh /usr/local / && rm /install_prereqs.sh
+COPY ./tools/requirements-py.txt / 
+RUN /install_prereqs.sh /usr/local / && rm /install_prereqs.sh && rm /requirements-py.txt
 RUN test -f /usr/local/hash || exit 1
 
 # Install yapf

diff --git a/tools/install_prereqs.sh b/tools/install_prereqs.sh
@@ -66,7 +66,7 @@ pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
 
 
 PREFIX=/usr/local
-REQ_FILE_DIR=""
+REQ_FILE_DIR="$(dirname -- "$0")"
 if [ "$#" -ge 2 ]; then
   PREFIX=$1
   cd $2
@@ -81,9 +81,5 @@ rbuild prepare -d $PREFIX -s develop
 
 if [[ ("${ID}" != "sles") ]]; then
 export CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=ON"
-
-pip3 install onnx==1.14.1 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0
-
-# pin version of protobuf in Python for onnx runtime unit tests between dist versions
-pip3 install protobuf==3.20.2
+pip3 install -r $REQ_FILE_DIR/requirements-py.txt
 fi
diff --git a/tools/requirements-py.txt b/tools/requirements-py.txt
@@ -0,0 +1,32 @@
+#####################################################################################
+# The MIT License (MIT)
+#
+# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#####################################################################################
+onnx==1.16;python_version>="3.11"
+onnx==1.14.1;python_version<"3.11"
+numpy==1.26.4;python_version>="3.11"
+numpy==1.21.6;python_version<"3.11"
+typing==3.7.4 
+pytest==6.0.1 
+packaging==23.0
+# pin version of protobuf in Python for onnx runtime unit tests between dist versions
+protobuf==3.20.2