diff --git a/docs/release-notes.md b/docs/release-notes.md
index 45fc5ba21..f7235a3ab 100644
--- a/docs/release-notes.md
+++ b/docs/release-notes.md
@@ -7,6 +7,9 @@
   * The installer now deletes Error Handler automatically if it's at the default path.
   * Updated mod compatibility list.
 
+* For mod authors:
+  * Fixed world-changed events (e.g. `ObjectListChanged`) not working correctly inside freshly-constructed building.
+
 ## 4.0.4
 Released 29 March 2024 for Stardew Valley 1.6.0 or later.
 
diff --git a/src/SMAPI/Framework/StateTracking/WorldLocationsTracker.cs b/src/SMAPI/Framework/StateTracking/WorldLocationsTracker.cs
index 8684444b9..e6ff933b5 100644
--- a/src/SMAPI/Framework/StateTracking/WorldLocationsTracker.cs
+++ b/src/SMAPI/Framework/StateTracking/WorldLocationsTracker.cs
@@ -102,14 +102,16 @@ public void Update()
             }
 
             // detect building interiors changed (e.g. construction completed)
-            foreach ((Building building, GameLocation? oldIndoors) in this.BuildingIndoors.Where(p => !object.Equals(p.Key.indoors.Value, p.Value)))
+            foreach ((Building building, GameLocation? oldIndoors) in this.BuildingIndoors)
             {
                 GameLocation? newIndoors = building.indoors.Value;
+                if (object.ReferenceEquals(oldIndoors, newIndoors))
+                    continue;
 
-                if (oldIndoors != null)
-                    this.Added.Add(oldIndoors);
-                if (newIndoors != null)
-                    this.Removed.Add(newIndoors);
+                this.Remove(oldIndoors);
+                this.Add(newIndoors);
+
+                this.BuildingIndoors[building] = newIndoors;
             }
         }