From f16786579c00aa986a61904cbe3c7bdb66ff0fc2 Mon Sep 17 00:00:00 2001
From: Abhishek Jindal <abjindal@microsoft.com>
Date: Fri, 25 Oct 2024 11:15:13 -0700
Subject: [PATCH] Fix CI pipeline (#1012)

Fix CI pipeline and install packages sequentially instead in single
requirements file. This might have been because of some conflict in
dependencies.
---
 .github/workflows/linux-cpu-x64-build.yml      |  5 +++--
 .../workflows/linux-cpu-x64-nightly-build.yml  |  3 ++-
 .github/workflows/linux-gpu-x64-build.yml      |  3 ++-
 .github/workflows/mac-cpu-arm64-build.yml      |  3 ++-
 .github/workflows/win-cpu-x64-build.yml        |  3 ++-
 .github/workflows/win-cuda-x64-build.yml       |  3 ++-
 .github/workflows/win-directml-x64-build.yml   |  3 ++-
 .pipelines/stages/jobs/py-validation-job.yml   | 18 ++++++++++++------
 test/python/cpu/ort/requirements.txt           |  2 ++
 test/python/cpu/torch/requirements.txt         |  2 ++
 .../ort/requirements.txt}                      |  2 --
 test/python/cuda/torch/requirements.txt        |  2 ++
 .../ort/requirements.txt}                      |  2 --
 test/python/directml/torch/requirements.txt    |  2 ++
 test/python/macos/ort/requirements.txt         |  2 ++
 test/python/macos/torch/requirements.txt       |  2 ++
 test/python/requirements-cpu.txt               |  4 ----
 test/python/requirements-macos.txt             |  4 ----
 18 files changed, 39 insertions(+), 26 deletions(-)
 create mode 100644 test/python/cpu/ort/requirements.txt
 create mode 100644 test/python/cpu/torch/requirements.txt
 rename test/python/{requirements-cuda.txt => cuda/ort/requirements.txt} (64%)
 create mode 100644 test/python/cuda/torch/requirements.txt
 rename test/python/{requirements-directml.txt => directml/ort/requirements.txt} (65%)
 create mode 100644 test/python/directml/torch/requirements.txt
 create mode 100644 test/python/macos/ort/requirements.txt
 create mode 100644 test/python/macos/torch/requirements.txt
 delete mode 100644 test/python/requirements-cpu.txt
 delete mode 100644 test/python/requirements-macos.txt

diff --git a/.github/workflows/linux-cpu-x64-build.yml b/.github/workflows/linux-cpu-x64-build.yml
index 8141bd27c..061885e3a 100644
--- a/.github/workflows/linux-cpu-x64-build.yml
+++ b/.github/workflows/linux-cpu-x64-build.yml
@@ -68,8 +68,9 @@ jobs:
       - name: Install the python wheel and test dependencies
         run: |
           python3 -m pip install -r test/python/requirements.txt --user
-          python3 -m pip install -r test/python/requirements-cpu.txt --user
-          python3 -m pip install build/cpu/wheel/onnxruntime_genai*.whl --user --no-deps
+          python3 -m pip install -r test/python/cpu/torch/requirements.txt --user
+          python3 -m pip install -r test/python/cpu/ort/requirements.txt --user
+          python3 -m pip install --user --no-index --find-links build/cpu/wheel onnxruntime_genai
 
       - name: Use Dummy HuggingFace Token
         run: |
diff --git a/.github/workflows/linux-cpu-x64-nightly-build.yml b/.github/workflows/linux-cpu-x64-nightly-build.yml
index 65573778c..61be5eb6f 100644
--- a/.github/workflows/linux-cpu-x64-nightly-build.yml
+++ b/.github/workflows/linux-cpu-x64-nightly-build.yml
@@ -51,7 +51,8 @@ jobs:
       - name: Install the python wheel and test dependencies
         run: |
           python3 -m pip install -r test/python/requirements.txt --user
-          python3 -m pip install -r test/python/requirements-cpu.txt --user
+          python3 -m pip install -r test/python/cpu/torch/requirements.txt --user
+          python3 -m pip install -r test/python/cpu/ort/requirements.txt --user
           python3 -m pip install build/cpu/wheel/onnxruntime_genai*.whl --no-deps
 
       - name: Use Dummy HuggingFace Token
diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml
index 040ddb712..50f94157b 100644
--- a/.github/workflows/linux-gpu-x64-build.yml
+++ b/.github/workflows/linux-gpu-x64-build.yml
@@ -124,7 +124,8 @@ jobs:
             -e HF_TOKEN=$HF_TOKEN \
             -w /ort_genai_src onnxruntimecudabuildx64 bash -c " \
               ${{ env.PYTHON_EXECUTABLE }} -m pip install -r test/python/requirements.txt --user && \
-              ${{ env.PYTHON_EXECUTABLE }} -m pip install -r test/python/requirements-cuda.txt --user && \
+              ${{ env.PYTHON_EXECUTABLE }} -m pip install -r test/python/cuda/torch/requirements.txt --user && \
+              ${{ env.PYTHON_EXECUTABLE }} -m pip install -r test/python/cuda/ort/requirements.txt --user && \
               ${{ env.PYTHON_EXECUTABLE }} -m pip install /ort_genai_src/build/cuda/wheel/onnxruntime_genai*manylinux*.whl --no-deps --user && \
               ${{ env.PYTHON_EXECUTABLE }} test/python/test_onnxruntime_genai.py --cwd test/python --test_models test/test_models --e2e"
 
diff --git a/.github/workflows/mac-cpu-arm64-build.yml b/.github/workflows/mac-cpu-arm64-build.yml
index 73bba3744..c37c5fa0f 100644
--- a/.github/workflows/mac-cpu-arm64-build.yml
+++ b/.github/workflows/mac-cpu-arm64-build.yml
@@ -54,7 +54,8 @@ jobs:
           python3 -m venv genai-macos-venv
           source genai-macos-venv/bin/activate
           python3 -m pip install -r test/python/requirements.txt
-          python3 -m pip install -r test/python/requirements-macos.txt
+          python3 -m pip install -r test/python/macos/torch/requirements.txt
+          python3 -m pip install -r test/python/macos/ort/requirements.txt
           python3 -m pip install build/cpu/osx-arm64/wheel/onnxruntime_genai*.whl --no-deps
 
       - name: Remove the ort lib and header files
diff --git a/.github/workflows/win-cpu-x64-build.yml b/.github/workflows/win-cpu-x64-build.yml
index 131b58862..22399ed89 100644
--- a/.github/workflows/win-cpu-x64-build.yml
+++ b/.github/workflows/win-cpu-x64-build.yml
@@ -75,7 +75,8 @@ jobs:
     - name: Install the python wheel and test dependencies
       run: |
         python3 -m pip install -r test\python\requirements.txt --user
-        python3 -m pip install -r test\python\requirements-cpu.txt --user
+        python3 -m pip install -r test\python\cpu\torch\requirements.txt --user
+        python3 -m pip install -r test\python\cpu\ort\requirements.txt --user
         python3 -m pip install (Get-ChildItem ("$env:binaryDir\wheel\*.whl")) --no-deps
 
     - name: Use Dummy HuggingFace Token
diff --git a/.github/workflows/win-cuda-x64-build.yml b/.github/workflows/win-cuda-x64-build.yml
index bd7d53e3b..3bff048a7 100644
--- a/.github/workflows/win-cuda-x64-build.yml
+++ b/.github/workflows/win-cuda-x64-build.yml
@@ -81,7 +81,8 @@ jobs:
     - name: Install the Python Wheel and Test Dependencies
       run: |
         python -m pip install -r test\python\requirements.txt
-        python -m pip install -r test\python\requirements-cuda.txt
+        python -m pip install -r test\python\cuda\torch\requirements.txt
+        python -m pip install -r test\python\cuda\ort\requirements.txt
         python -m pip install (Get-ChildItem ("$env:binaryDir\wheel\*.whl")) --no-deps
 
     - name: Use Dummy HuggingFace Token
diff --git a/.github/workflows/win-directml-x64-build.yml b/.github/workflows/win-directml-x64-build.yml
index 53cf74ed3..678573606 100644
--- a/.github/workflows/win-directml-x64-build.yml
+++ b/.github/workflows/win-directml-x64-build.yml
@@ -89,7 +89,8 @@ jobs:
     - name: Install the Python Wheel and Test Dependencies
       run: |
         python -m pip install -r test\python\requirements.txt
-        python -m pip install -r test\python\requirements-directml.txt
+        python -m pip install -r test\python\directml\torch\requirements.txt
+        python -m pip install -r test\python\directml\ort\requirements.txt
         python -m pip install (Get-ChildItem ("$env:binaryDir\wheel\*.whl")) --no-deps
 
     - name: Verify Build Artifacts
diff --git a/.pipelines/stages/jobs/py-validation-job.yml b/.pipelines/stages/jobs/py-validation-job.yml
index 2282f8f77..8657bab66 100644
--- a/.pipelines/stages/jobs/py-validation-job.yml
+++ b/.pipelines/stages/jobs/py-validation-job.yml
@@ -188,7 +188,8 @@ jobs:
             bash -c " \
                 export ORTGENAI_LOG_ORT_LIB=1 && \
                 $python_exe -m pip install -r /ort_genai_src/test/python/requirements.txt && \
-                $python_exe -m pip install -r /ort_genai_src/test/python/requirements-cuda.txt && \
+                $python_exe -m pip install -r /ort_genai_src/test/python/cuda/torch/requirements.txt && \
+                $python_exe -m pip install -r /ort_genai_src/test/python/cuda/ort/requirements.txt && \
                 cd /ort_genai_src/examples/python && \
                 $python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
                 $python_exe model-generate.py -m ./models/$(prebuild_phi3_mini_model_folder) --min_length 25 --max_length 50 --verbose"
@@ -200,10 +201,12 @@ jobs:
           export ORTGENAI_LOG_ORT_LIB=1
           python -m pip install -r test/python/requirements.txt
           if [[ "$(os)" == "linux" ]]; then
-            python -m pip install -r test/python/requirements-cpu.txt
+            python -m pip install -r test/python/cpu/torch/requirements.txt
+            python -m pip install -r test/python/cpu/ort/requirements.txt
           fi
           if [[ "$(os)" == "osx" ]]; then
-            python -m pip install -r test/python/requirements-macos.txt
+            python -m pip install -r test/python/macos/torch/requirements.txt
+            python -m pip install -r test/python/macos/ort/requirements.txt
           fi
           cd examples/python
           python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
@@ -224,13 +227,16 @@ jobs:
           $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
           $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
           Write-Host $env:PATH
-          python -m pip install -r test/python/requirements-cuda.txt
+          python -m pip install -r test/python/cuda/torch/requirements.txt
+          python -m pip install -r test/python/cuda/ort/requirements.txt
         }
         elseif ("$(ep)" -eq "directml") {
-          python -m pip install -r test/python/requirements-directml.txt
+          python -m pip install -r test/python/requirements-torch-directml.txt
+          python -m pip install -r test/python/requirements-torch-directml.txt
         }
         else {
-          python -m pip install -r test/python/requirements-cpu.txt
+          python -m pip install -r test/python/cpu/torch/requirements.txt
+          python -m pip install -r test/python/cpu/ort/requirements.txt
         }
         cd examples\python
         python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
diff --git a/test/python/cpu/ort/requirements.txt b/test/python/cpu/ort/requirements.txt
new file mode 100644
index 000000000..c00054ea3
--- /dev/null
+++ b/test/python/cpu/ort/requirements.txt
@@ -0,0 +1,2 @@
+-i https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/
+onnxruntime==1.20.0.dev20241017001
diff --git a/test/python/cpu/torch/requirements.txt b/test/python/cpu/torch/requirements.txt
new file mode 100644
index 000000000..4f7c53dc4
--- /dev/null
+++ b/test/python/cpu/torch/requirements.txt
@@ -0,0 +1,2 @@
+-f https://download.pytorch.org/whl/torch_stable.html
+torch==2.2.1+cpu
diff --git a/test/python/requirements-cuda.txt b/test/python/cuda/ort/requirements.txt
similarity index 64%
rename from test/python/requirements-cuda.txt
rename to test/python/cuda/ort/requirements.txt
index 68370c166..db83e9abb 100644
--- a/test/python/requirements-cuda.txt
+++ b/test/python/cuda/ort/requirements.txt
@@ -1,4 +1,2 @@
--f https://download.pytorch.org/whl/torch_stable.html
 -i https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/
-torch==2.2.1+cu121
 ort-nightly-gpu==1.20.0.dev20241007001
diff --git a/test/python/cuda/torch/requirements.txt b/test/python/cuda/torch/requirements.txt
new file mode 100644
index 000000000..04fa85e79
--- /dev/null
+++ b/test/python/cuda/torch/requirements.txt
@@ -0,0 +1,2 @@
+-f https://download.pytorch.org/whl/torch_stable.html
+torch==2.2.1+cu121
diff --git a/test/python/requirements-directml.txt b/test/python/directml/ort/requirements.txt
similarity index 65%
rename from test/python/requirements-directml.txt
rename to test/python/directml/ort/requirements.txt
index 87a94a44c..5b4d86218 100644
--- a/test/python/requirements-directml.txt
+++ b/test/python/directml/ort/requirements.txt
@@ -1,4 +1,2 @@
--f https://download.pytorch.org/whl/torch_stable.html
 -i https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/
-torch==2.2.1+cpu
 ort-nightly-directml==1.20.0.dev20241006001
diff --git a/test/python/directml/torch/requirements.txt b/test/python/directml/torch/requirements.txt
new file mode 100644
index 000000000..4f7c53dc4
--- /dev/null
+++ b/test/python/directml/torch/requirements.txt
@@ -0,0 +1,2 @@
+-f https://download.pytorch.org/whl/torch_stable.html
+torch==2.2.1+cpu
diff --git a/test/python/macos/ort/requirements.txt b/test/python/macos/ort/requirements.txt
new file mode 100644
index 000000000..c00054ea3
--- /dev/null
+++ b/test/python/macos/ort/requirements.txt
@@ -0,0 +1,2 @@
+-i https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/
+onnxruntime==1.20.0.dev20241017001
diff --git a/test/python/macos/torch/requirements.txt b/test/python/macos/torch/requirements.txt
new file mode 100644
index 000000000..7f1670fdd
--- /dev/null
+++ b/test/python/macos/torch/requirements.txt
@@ -0,0 +1,2 @@
+-f https://download.pytorch.org/whl/torch_stable.html
+torch==2.2.1
diff --git a/test/python/requirements-cpu.txt b/test/python/requirements-cpu.txt
deleted file mode 100644
index 8bbb0eec6..000000000
--- a/test/python/requirements-cpu.txt
+++ /dev/null
@@ -1,4 +0,0 @@
--f https://download.pytorch.org/whl/torch_stable.html
--i https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/
-torch==2.2.1+cpu
-ort-nightly==1.20.0.dev20241006001
diff --git a/test/python/requirements-macos.txt b/test/python/requirements-macos.txt
deleted file mode 100644
index 7f557ce5a..000000000
--- a/test/python/requirements-macos.txt
+++ /dev/null
@@ -1,4 +0,0 @@
--f https://download.pytorch.org/whl/torch_stable.html
--i https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/
-torch==2.2.1
-ort-nightly==1.20.0.dev20241006001