diff --git a/openapi31/entities.go b/openapi31/entities.go
index 1b79855..5b4c7e7 100644
--- a/openapi31/entities.go
+++ b/openapi31/entities.go
@@ -1187,6 +1187,13 @@ func (r *Reference) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
+// IsZero tells if the object has no value set.
+func (h *Reference) IsZero() bool {
+	return h.Ref == "" &&
+		(h.Summary == nil || *h.Summary == "") &&
+		(h.Description == nil || *h.Description == "")
+}
+
 // Parameter structure is generated from "#/$defs/parameter".
 type Parameter struct {
 	Name          string                        `json:"name"` // Required.
@@ -1727,6 +1734,19 @@ func (h *Header) WithMapOfAnythingItem(key string, val interface{}) *Header {
 	return h
 }
 
+// IsZero tells if the object has no value set.
+func (h *Header) IsZero() bool {
+	return (h.Description == nil || *h.Description == "") &&
+		h.Required == nil &&
+		h.Deprecated == nil &&
+		(h.Schema == nil || len(h.Schema) == 0) &&
+		(h.Content == nil || len(h.Content) == 0) &&
+		(h.Example == nil) &&
+		(h.Examples == nil || len(h.Examples) == 0) &&
+		h.Explode == nil &&
+		(h.MapOfAnything == nil || len(h.MapOfAnything) == 0)
+}
+
 type marshalHeader Header
 
 var knownKeysHeader = []string{
@@ -2061,7 +2081,7 @@ func (h *HeaderOrReference) UnmarshalJSON(data []byte) error {
 	if err != nil {
 		oneOfErrors["Reference"] = err
 		h.Reference = nil
-	} else {
+	} else if !h.Reference.IsZero() {
 		oneOfValid++
 	}
 
@@ -2069,7 +2089,7 @@ func (h *HeaderOrReference) UnmarshalJSON(data []byte) error {
 	if err != nil {
 		oneOfErrors["Header"] = err
 		h.Header = nil
-	} else {
+	} else if !h.Header.IsZero() {
 		oneOfValid++
 	}
 
diff --git a/openapi31/entities_test.go b/openapi31/entities_test.go
index 759a621..045fef1 100644
--- a/openapi31/entities_test.go
+++ b/openapi31/entities_test.go
@@ -15,3 +15,44 @@ func TestSpec_UnmarshalYAML(t *testing.T) {
 	refl := openapi31.NewReflector()
 	require.NoError(t, refl.Spec.UnmarshalYAML(bytes))
 }
+
+func TestSpec_UnmarshalYAML_refsInResponseHeaders(t *testing.T) {
+	var s openapi31.Spec
+
+	spec := `openapi: 3.1.0f
+info:
+  description: description
+  license:
+    name: Apache-2.0
+    url: https://www.apache.org/licenses/LICENSE-2.0.html
+  title: title
+  version: 2.0.0
+servers:
+  - url: /v2
+paths:
+  /user:
+    put:
+      summary: updates the user by id
+      operationId: UpdateUser
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: string
+        description: Updated user object
+        required: true
+      responses:
+        "404":
+          description: User not found
+          headers:
+            Cache-Control:
+              $ref: '#/components/headers/CacheControl'
+components:
+  headers:
+    CacheControl:
+      schema:
+        type: string
+`
+
+	require.NoError(t, s.UnmarshalYAML([]byte(spec)))
+}