diff --git a/README.md b/README.md
index 63bf96d..e789601 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,6 @@
 # Dockerfiles
 
+## Cache Strategy
 
 ## Deprecations/Removals
 
diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py
index 8d104ba..c7ad06e 100644
--- a/pants-plugins/macros.py
+++ b/pants-plugins/macros.py
@@ -1,3 +1,39 @@
+def default_ghcr_cache(target_name: str) -> tuple[str, Optional[str]]:  # noqa: F821
+    """
+    From Github Actions default env vars, try to constitute
+    https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
+    """
+    # Need to specify this, because Pants doesn't reconcile `@ghcr` in the cache details
+    REGISTRY = "ghcr.io/robotpajamas"
+    REPO_NAME = "dockerfiles"
+
+    # These are always populated in a PR, but not in a direct push
+    head_ref: str = env("GITHUB_HEAD_REF", "")  # noqa: F821
+    base_ref: str = env("GITHUB_BASE_REF", "")  # noqa: F821
+
+    # This is always populated, but in a PR it is: <pr_number>/merge - if we're in a PR, ignore
+    ref_name: str = env("GITHUB_REF_NAME", "") if not head_ref else ""  # noqa: F821
+
+    cache_from_refs = [
+        f"{REGISTRY}/{REPO_NAME}/{target_name}/build-cache:{tag.replace('/', '-')}"
+        for tag in [head_ref, base_ref, ref_name, "main"]
+        if tag
+    ]
+    cache_from = [{"type": "registry", "ref": ref} for ref in cache_from_refs]
+
+    # Cache to the head_ref only in a PR, otherwise, cache to the ref_name
+    resolved_ref = head_ref or ref_name
+    cache_to = None
+    if resolved_ref:
+        cache_to = {
+            "type": "registry",
+            "mode": "max",
+            "ref": f"{REGISTRY}/{REPO_NAME}/{target_name}/build-cache:{resolved_ref.replace('/', '-')}",
+        }
+
+    return cache_from, cache_to
+
+
 def docker_image_gha(name: str, **kwargs) -> None:
     """
     A macro to create a Docker image that caches to Github Actions, by default.
@@ -12,20 +48,14 @@ def docker_image_gha(name: str, **kwargs) -> None:
     #     **kwargs,
     # )
 
+    cache_from, cache_to = default_ghcr_cache(name)
     cache_from = kwargs.pop(
         "cache_from",
-        {
-            "type": "registry",
-            "ref": f"ghcr.io/robotpajamas/dockerfiles/{name}/build-cache:main",
-        },
+        cache_from,
     )
     cache_to = kwargs.pop(
         "cache_to",
-        {
-            "type": "registry",
-            "mode": "max",
-            "ref": f"ghcr.io/robotpajamas/dockerfiles/{name}/build-cache:main",
-        },
+        cache_to,
     )
 
     docker_image(  # noqa: F821
diff --git a/pants.ci.toml b/pants.ci.toml
index c65cb0f..16d425f 100644
--- a/pants.ci.toml
+++ b/pants.ci.toml
@@ -8,3 +8,6 @@ env_vars = [
   "ACTIONS_CACHE_URL",
   "ACTIONS_RUNTIME_TOKEN",
 ]
+
+[docker.registries.ghcr]
+address = "ghcr.io/robotpajamas"