diff --git a/controller/controlplane/controller_watch.go b/controller/controlplane/controller_watch.go
index 9271ed199..c231efb71 100644
--- a/controller/controlplane/controller_watch.go
+++ b/controller/controlplane/controller_watch.go
@@ -175,13 +175,11 @@ func objectIsOwnedByControlPlane(obj client.Object, cp *operatorv1beta1.ControlP
 		return true
 	}
 
-	if labels := obj.GetLabels(); len(labels) > 0 {
-		if labels[consts.GatewayOperatorManagedByNameLabel] == cp.Name {
-			if obj.GetNamespace() != "" {
-				return cp.GetNamespace() == obj.GetNamespace()
-			} else {
-				return true
-			}
+	if labels[consts.GatewayOperatorManagedByNameLabel] == cp.Name {
+		if obj.GetNamespace() != "" {
+			return cp.GetNamespace() == obj.GetNamespace()
+		} else {
+			return true
 		}
 	}