diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0a1d3d4..77f3965 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -160,6 +160,7 @@ endif() # TRITON_PYTORCH_ENABLE_TORCHTRT
 
 if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
   set(LIBS_ARCH "aarch64")
+  set(CUSPARSE_ARCH "sbsa")
   set(LIBTORCH_LIBS
       "libopenblas.so.0"
       "libnvpl_blas_core.so.0"
@@ -175,6 +176,7 @@ if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
   )
 else()
   set(LIBS_ARCH "x86_64")
+  set(CUSPARSE_ARCH "x86_64")
   set(LIBTORCH_LIBS
     "libmkl_avx2.so.1"
     "libmkl_avx512.so.1"
@@ -230,7 +232,7 @@ if (${TRITON_PYTORCH_DOCKER_BUILD})
     COMMAND docker cp pytorch_backend_ptlib:/usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_global_deps.so libtorch_global_deps.so
     COMMAND docker cp pytorch_backend_ptlib:/usr/local/lib/python3.10/dist-packages/torch/lib/libcaffe2_nvrtc.so libcaffe2_nvrtc.so
     # TODO: Revisit when not needed by making it part of cuda base container.
-    COMMAND docker cp -L pytorch_backend_ptlib:/usr/local/cuda-12.5/targets/${LIBS_ARCH}-linux/lib/libcusparseLt.so libcusparseLt.so
+    COMMAND docker cp -L pytorch_backend_ptlib:/usr/local/cuda-12.5/targets/${CUSPARSE_ARCH}-linux/lib/libcusparseLt.so libcusparseLt.so
     COMMAND docker cp pytorch_backend_ptlib:/usr/local/lib/libtorchvision.so libtorchvision.so
     COMMAND /bin/sh -c "if [ ${TRITON_PYTORCH_ENABLE_TORCHTRT} = 'ON' ]; then docker cp pytorch_backend_ptlib:/usr/local/lib/python3.10/dist-packages/torch_tensorrt/lib/libtorchtrt_runtime.so libtorchtrt_runtime.so; fi"
     COMMAND docker cp pytorch_backend_ptlib:/usr/local/lib/python3.10/dist-packages/torch_tensorrt/bin/torchtrtc torchtrtc || echo "error ignored..." || true