diff --git a/pkg/image/build.go b/pkg/image/build.go
index 74f4f6d126..db2a1cf850 100644
--- a/pkg/image/build.go
+++ b/pkg/image/build.go
@@ -51,7 +51,7 @@ func Build(cfg *config.Config, dir, imageName string, secrets []string, noCache,
 			return fmt.Errorf("Failed to generate weights manifest: %w", err)
 		}
 		cachedManifest, err := weights.LoadManifest(weightsManifestPath)
-		changed := err != nil && weightsManifest.Equal(cachedManifest)
+		changed := err != nil || !weightsManifest.Equal(cachedManifest)
 		if changed {
 			if err := buildWeightsImage(dir, weightsDockerfile, imageName+"-weights", secrets, noCache, progressOutput); err != nil {
 				return fmt.Errorf("Failed to build model weights Docker image: %w", err)