diff --git a/lib/ruby_lsp/tapioca/addon.rb b/lib/ruby_lsp/tapioca/addon.rb
index eef5fa80b..2923b915d 100644
--- a/lib/ruby_lsp/tapioca/addon.rb
+++ b/lib/ruby_lsp/tapioca/addon.rb
@@ -47,7 +47,9 @@ def activate(global_state, outgoing_queue)
           @outgoing_queue << Notification.window_log_message("Activating Tapioca add-on v#{version}")
           @rails_runner_client.register_server_addon(File.expand_path("server_addon.rb", __dir__))
 
-          generate_gem_rbis if git_repo? && lockfile_changed?
+          if git_repo?
+            lockfile_changed? ? generate_gem_rbis : cleanup_orphaned_rbis
+          end
         rescue IncompatibleApiError
           # The requested version for the Rails add-on no longer matches. We need to upgrade and fix the breaking
           # changes
@@ -154,6 +156,22 @@ def generate_gem_rbis
           diff: T.must(@lockfile_diff),
         )
       end
+
+      sig { void }
+      def cleanup_orphaned_rbis
+        untracked_files = %x(git ls-files --others --exclude-standard sorbet/rbi/gems/).lines.map(&:strip)
+        deleted_files = %x(git ls-files --deleted sorbet/rbi/gems/).lines.map(&:strip)
+
+        untracked_files.each do |file|
+          File.delete(file)
+          $stderr.puts "Deleted untracked RBI: #{file}"
+        end
+
+        deleted_files.each do |file|
+          %x(git checkout -- #{file})
+          $stderr.puts "Restored deleted RBI: #{file}"
+        end
+      end
     end
   end
 end