diff --git a/.github/workflows/build_conda_linux.yml b/.github/workflows/build_conda_linux.yml
index 81fb4b066..76d3b526d 100644
--- a/.github/workflows/build_conda_linux.yml
+++ b/.github/workflows/build_conda_linux.yml
@@ -16,6 +16,7 @@ jobs:
       test-infra-repository: pytorch/test-infra
       test-infra-ref: main
       with-cuda: disable
+      with-rocm: disable
   build:
     needs: generate-matrix
     strategy:
diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml
index cfb28f1b0..1f1c9d9ef 100644
--- a/.github/workflows/build_wheels_linux.yml
+++ b/.github/workflows/build_wheels_linux.yml
@@ -16,6 +16,7 @@ jobs:
       test-infra-repository: pytorch/test-infra
       test-infra-ref: main
       with-cuda: disable
+      with-rocm: disable
   build:
     needs: generate-matrix
     strategy:
diff --git a/setup.py b/setup.py
index 77ae89cb8..13cdc42e1 100644
--- a/setup.py
+++ b/setup.py
@@ -110,11 +110,7 @@ def _get_requirements():
 
 
 # Use new version of torch on main branch
-pytorch_package_dep = "torch>2.0"
-if os.getenv("PYTORCH_VERSION"):
-    pytorch_package_dep = pytorch_package_dep.split(">")[0]
-    pytorch_package_dep += "==" + os.getenv("PYTORCH_VERSION")
-
+pytorch_package_dep = "torch>=2"
 
 requirements = _get_requirements()
 requirements.append(pytorch_package_dep)