diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh
index bc49c80a309..4217a9d24be 100755
--- a/.github/scripts/cmake.sh
+++ b/.github/scripts/cmake.sh
@@ -30,6 +30,10 @@ else
   JOBS=$(nproc)
 fi
 
+if [[ $OS_TYPE == linux ]]; then
+  export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}"
+fi
+
 TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))")
 if [[ $OS_TYPE == windows ]]; then
   PACKAGING_DIR="${PWD}/packaging"
diff --git a/.github/workflows/build-cmake.yml b/.github/workflows/build-cmake.yml
index 4006c423dae..9cee3bfc26d 100644
--- a/.github/workflows/build-cmake.yml
+++ b/.github/workflows/build-cmake.yml
@@ -33,11 +33,6 @@ jobs:
         export PYTHON_VERSION=3.9
         export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }}
         export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }}
-        # Prepare conda
-        CONDA_PATH=$(which conda)
-        eval "$(${CONDA_PATH} shell.bash hook)"
-        conda activate ci
-        export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}"
         ./.github/scripts/cmake.sh
 
   macos: