diff --git a/vllm/utils.py b/vllm/utils.py
index 1331ebfd9600..9da8f1dbfb32 100644
--- a/vllm/utils.py
+++ b/vllm/utils.py
@@ -119,11 +119,13 @@ def is_hip() -> bool:
 
 @lru_cache(maxsize=None)
 def is_cpu() -> bool:
-    from importlib.metadata import version
+    from importlib.metadata import PackageNotFoundError, version
 
     # UPSTREAM SYNC: needed for nm-vllm
-    is_cpu_flag = "cpu" in version("nm-vllm")
-    return is_cpu_flag
+    try:
+        return "cpu" in version("nm-vllm")
+    except PackageNotFoundError:
+        return False
 
 
 @lru_cache(maxsize=None)