System: Add advanced 'Export Shared Memory' option

Memory map is exported as duckstation_<pid>. Previously, this only
worked on Windows, now it is extended to Linux as well.

src/common/crash_handler.cpp

@@ -85,6 +85,7 @@ static bool WriteMinidump(HMODULE hDbgHelp, HANDLE hFile, HANDLE hProcess, DWORD
 
 static std::wstring s_write_directory;
 static DynamicLibrary s_dbghelp_module;
+static CrashHandler::CleanupHandler s_cleanup_handler;
 static bool s_in_crash_handler = false;
 
 static void GenerateCrashFilename(wchar_t* buf, size_t len, const wchar_t* prefix, const wchar_t* extension)
@@ -99,8 +100,6 @@ static void GenerateCrashFilename(wchar_t* buf, size_t len, const wchar_t* prefi
 
 static void WriteMinidumpAndCallstack(PEXCEPTION_POINTERS exi)
 {
-  s_in_crash_handler = true;
-
   wchar_t filename[1024] = {};
   GenerateCrashFilename(filename, std::size(filename), s_write_directory.empty() ? nullptr : s_write_directory.c_str(),
                         L"txt");
@@ -148,15 +147,21 @@ static LONG NTAPI ExceptionHandler(PEXCEPTION_POINTERS exi)
 {
   // if the debugger is attached, or we're recursively crashing, let it take care of it.
   if (!s_in_crash_handler)
+  {
+    s_in_crash_handler = true;
+    if (s_cleanup_handler)
+      s_cleanup_handler();
+
     WriteMinidumpAndCallstack(exi);
+  }
 
   // returning EXCEPTION_CONTINUE_SEARCH makes sense, except for the fact that it seems to leave zombie processes
   // around. instead, force ourselves to terminate.
   TerminateProcess(GetCurrentProcess(), 0xFEFEFEFEu);
   return EXCEPTION_CONTINUE_SEARCH;
 }
 
-bool CrashHandler::Install()
+bool CrashHandler::Install(CleanupHandler cleanup_handler)
 {
   // load dbghelp at install/startup, that way we're not LoadLibrary()'ing after a crash
   // .. because that probably wouldn't go down well.
@@ -165,6 +170,7 @@ bool CrashHandler::Install()
     s_dbghelp_module.Adopt(mod);
 
   SetUnhandledExceptionFilter(ExceptionHandler);
+  s_cleanup_handler = cleanup_handler;
   return true;
 }
 
@@ -208,6 +214,7 @@ static void LogCallstack(int signal, const void* exception_pc);
 static std::recursive_mutex s_crash_mutex;
 static bool s_in_signal_handler = false;
 
+static CleanupHandler s_cleanup_handler;
 static backtrace_state* s_backtrace_state = nullptr;
 } // namespace CrashHandler
 
@@ -304,6 +311,9 @@ void CrashHandler::CrashSignalHandler(int signal, siginfo_t* siginfo, void* ctx)
   {
     s_in_signal_handler = true;
 
+    if (s_cleanup_handler)
+      s_cleanup_handler();
+
 #if defined(__APPLE__) && defined(__x86_64__)
     void* const exception_pc = reinterpret_cast<void*>(static_cast<ucontext_t*>(ctx)->uc_mcontext->__ss.__rip);
 #elif defined(__FreeBSD__) && defined(__x86_64__)
@@ -327,7 +337,7 @@ void CrashHandler::CrashSignalHandler(int signal, siginfo_t* siginfo, void* ctx)
   std::abort();
 }
 
-bool CrashHandler::Install()
+bool CrashHandler::Install(CleanupHandler cleanup_handler)
 {
   const std::string progpath = FileSystem::GetProgramPath();
   s_backtrace_state = backtrace_create_state(progpath.empty() ? nullptr : progpath.c_str(), 0, nullptr, nullptr);
@@ -344,6 +354,7 @@ bool CrashHandler::Install()
   if (sigaction(SIGSEGV, &sa, nullptr) != 0)
     return false;
 
+  s_cleanup_handler = cleanup_handler;
   return true;
 }
 

src/common/crash_handler.h

@@ -1,21 +1,30 @@
 // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <[email protected]>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
-#include "types.h"
+#pragma once
+
 #include <string_view>
 
 #ifndef _WIN32
 #include <signal.h>
 #endif
 
 namespace CrashHandler {
-bool Install();
+
+/// Adds a callback to run just before the crash handler exits.
+/// It's not guaranteed that this handler will actually run, because the process state could be very messed up by this
+/// point. It's mainly a thing so that we can free up the shared memory object if there was one created.
+using CleanupHandler = void(*)();
+
+bool Install(CleanupHandler cleanup_handler);
 void SetWriteDirectory(std::string_view dump_directory);
 void WriteDumpForCaller();
 
 #ifndef _WIN32
+
 // Allow crash handler to be invoked from a signal.
 void CrashSignalHandler(int signal, siginfo_t* siginfo, void* ctx);
+
 #endif
 
 } // namespace CrashHandler

src/common/memmap.cpp

@@ -66,9 +66,10 @@ std::string MemMap::GetFileMappingName(const char* prefix)
 
 void* MemMap::CreateSharedMemory(const char* name, size_t size, Error* error)
 {
+  const std::wstring mapping_name = name ? StringUtil::UTF8StringToWideString(name) : std::wstring();
   const HANDLE mapping =
     CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, static_cast<DWORD>(size >> 32),
-                       static_cast<DWORD>(size), StringUtil::UTF8StringToWideString(name).c_str());
+                       static_cast<DWORD>(size), mapping_name.empty() ? nullptr : mapping_name.c_str());
   if (!mapping)
     Error::SetWin32(error, "CreateFileMappingW() failed: ", GetLastError());
 
@@ -80,6 +81,11 @@ void MemMap::DestroySharedMemory(void* ptr)
   CloseHandle(static_cast<HANDLE>(ptr));
 }
 
+void MemMap::DeleteSharedMemory(const char* name)
+{
+  // Automatically freed on close.
+}
+
 void* MemMap::MapSharedMemory(void* handle, size_t offset, void* baseaddr, size_t size, PageProtect mode)
 {
   void* ret = MapViewOfFileEx(static_cast<HANDLE>(handle), FILE_MAP_READ | FILE_MAP_WRITE,
@@ -374,6 +380,10 @@ void MemMap::DestroySharedMemory(void* ptr)
   mach_port_deallocate(mach_task_self(), static_cast<mach_port_t>(reinterpret_cast<uintptr_t>(ptr)));
 }
 
+void MemMap::DeleteSharedMemory(const char* name)
+{
+}
+
 void* MemMap::MapSharedMemory(void* handle, size_t offset, void* baseaddr, size_t size, PageProtect mode)
 {
   mach_vm_address_t ptr = reinterpret_cast<mach_vm_address_t>(baseaddr);
@@ -617,15 +627,25 @@ std::string MemMap::GetFileMappingName(const char* prefix)
 
 void* MemMap::CreateSharedMemory(const char* name, size_t size, Error* error)
 {
+  const bool is_anonymous = (!name || *name == 0);
+#if defined(__linux__) || defined(__FreeBSD__)
+  const int fd = is_anonymous ? memfd_create("", 0) : shm_open(name, O_CREAT | O_EXCL | O_RDWR, 0600);
+  if (fd < 0)
+  {
+    Error::SetErrno(error, is_anonymous ? "memfd_create() failed: " : "shm_open() failed: ", errno);
+    return nullptr;
+  }
+#else
   const int fd = shm_open(name, O_CREAT | O_EXCL | O_RDWR, 0600);
   if (fd < 0)
   {
-    Error::SetErrno(error, "shm_open failed: ", errno);
+    Error::SetErrno(error, "shm_open() failed: ", errno);
     return nullptr;
   }
 
   // we're not going to be opening this mapping in other processes, so remove the file
   shm_unlink(name);
+#endif
 
   // use fallocate() to ensure we don't SIGBUS later on.
 #ifdef __linux__
@@ -651,6 +671,11 @@ void MemMap::DestroySharedMemory(void* ptr)
   close(static_cast<int>(reinterpret_cast<intptr_t>(ptr)));
 }
 
+void MemMap::DeleteSharedMemory(const char* name)
+{
+  shm_unlink(name);
+}
+
 void* MemMap::MapSharedMemory(void* handle, size_t offset, void* baseaddr, size_t size, PageProtect mode)
 {
   const int flags = (baseaddr != nullptr) ? (MAP_SHARED | MAP_FIXED) : MAP_SHARED;

src/common/memmap.h

@@ -53,6 +53,7 @@ class Error;
 namespace MemMap {
 std::string GetFileMappingName(const char* prefix);
 void* CreateSharedMemory(const char* name, size_t size, Error* error);
+void DeleteSharedMemory(const char* name);
 void DestroySharedMemory(void* ptr);
 void* MapSharedMemory(void* handle, size_t offset, void* baseaddr, size_t size, PageProtect mode);
 void UnmapSharedMemory(void* baseaddr, size_t size);

src/core/bus.cpp

@@ -118,6 +118,7 @@ union RAM_SIZE_REG
 } // namespace
 
 static void* s_shmem_handle = nullptr;
+static std::string s_shmem_name;
 
 std::bitset<RAM_8MB_CODE_PAGE_COUNT> g_ram_code_bits{};
 u8* g_ram = nullptr;
@@ -153,6 +154,8 @@ static u8** s_fastmem_lut = nullptr;
 
 static bool s_kernel_initialize_hook_run = false;
 
+static bool AllocateMemoryMap(Error* error);
+static void ReleaseMemoryMap();
 static void SetRAMSize(bool enable_8mb_ram);
 
 static std::tuple<TickCount, TickCount, TickCount> CalculateMemoryTiming(MEMDELAY mem_delay, COMDELAY common_delay);
@@ -194,10 +197,12 @@ static constexpr size_t TOTAL_SIZE = LUT_OFFSET + LUT_SIZE;
 #define FIXUP_WORD_WRITE_VALUE(size, offset, value)                                                                    \
   ((size == MemoryAccessSize::Word) ? (value) : ((value) << (((offset) & 3u) * 8)))
 
-bool Bus::AllocateMemory(Error* error)
+bool Bus::AllocateMemoryMap(Error* error)
 {
-  s_shmem_handle =
-    MemMap::CreateSharedMemory(MemMap::GetFileMappingName("duckstation").c_str(), MemoryMap::TOTAL_SIZE, error);
+  // This executes super early in process startup, therefore export_shared_memory will always be false.
+  if (g_settings.export_shared_memory)
+    s_shmem_name = MemMap::GetFileMappingName("duckstation");
+  s_shmem_handle = MemMap::CreateSharedMemory(s_shmem_name.c_str(), MemoryMap::TOTAL_SIZE, error);
   if (!s_shmem_handle)
   {
 #ifndef __linux__
@@ -216,7 +221,7 @@ bool Bus::AllocateMemory(Error* error)
   if (!g_ram || !g_unprotected_ram)
   {
     Error::SetStringView(error, "Failed to map memory for RAM");
-    ReleaseMemory();
+    ReleaseMemoryMap();
     return false;
   }
 
@@ -227,7 +232,7 @@ bool Bus::AllocateMemory(Error* error)
   if (!g_bios)
   {
     Error::SetStringView(error, "Failed to map memory for BIOS");
-    ReleaseMemory();
+    ReleaseMemoryMap();
     return false;
   }
 
@@ -238,48 +243,29 @@ bool Bus::AllocateMemory(Error* error)
   if (!g_memory_handlers)
   {
     Error::SetStringView(error, "Failed to map memory for LUTs");
-    ReleaseMemory();
+    ReleaseMemoryMap();
     return false;
   }
 
   VERBOSE_LOG("LUTs are mapped at {}.", static_cast<void*>(g_memory_handlers));
   g_memory_handlers_isc = g_memory_handlers + MEMORY_LUT_SLOTS;
   SetHandlers();
 
-#ifdef ENABLE_MMAP_FASTMEM
-  if (!s_fastmem_arena.Create(FASTMEM_ARENA_SIZE))
-  {
-    Error::SetStringView(error, "Failed to create fastmem arena");
-    ReleaseMemory();
-    return false;
-  }
-
-  INFO_LOG("Fastmem base: {}", static_cast<void*>(s_fastmem_arena.BasePointer()));
-#endif
-
 #ifndef __ANDROID__
   Exports::RAM = reinterpret_cast<uintptr_t>(g_unprotected_ram);
 #endif
 
   return true;
 }
 
-void Bus::ReleaseMemory()
+void Bus::ReleaseMemoryMap()
 {
 #ifndef __ANDROID__
   Exports::RAM = 0;
   Exports::RAM_SIZE = 0;
   Exports::RAM_MASK = 0;
 #endif
 
-#ifdef ENABLE_MMAP_FASTMEM
-  DebugAssert(s_fastmem_ram_views.empty());
-  s_fastmem_arena.Destroy();
-#endif
-
-  std::free(s_fastmem_lut);
-  s_fastmem_lut = nullptr;
-
   g_memory_handlers_isc = nullptr;
   if (g_memory_handlers)
   {
@@ -309,7 +295,86 @@ void Bus::ReleaseMemory()
   {
     MemMap::DestroySharedMemory(s_shmem_handle);
     s_shmem_handle = nullptr;
+
+    if (!s_shmem_name.empty())
+    {
+      MemMap::DeleteSharedMemory(s_shmem_name.c_str());
+      s_shmem_name = {};
+    }
+  }
+}
+
+bool Bus::AllocateMemory(Error* error)
+{
+  if (!AllocateMemoryMap(error))
+    return false;
+
+#ifdef ENABLE_MMAP_FASTMEM
+  if (!s_fastmem_arena.Create(FASTMEM_ARENA_SIZE))
+  {
+    Error::SetStringView(error, "Failed to create fastmem arena");
+    ReleaseMemory();
+    return false;
+  }
+
+  INFO_LOG("Fastmem base: {}", static_cast<void*>(s_fastmem_arena.BasePointer()));
+#endif
+
+  return true;
+}
+
+void Bus::ReleaseMemory()
+{
+#ifdef ENABLE_MMAP_FASTMEM
+  DebugAssert(s_fastmem_ram_views.empty());
+  s_fastmem_arena.Destroy();
+#endif
+
+  std::free(s_fastmem_lut);
+  s_fastmem_lut = nullptr;
+
+  ReleaseMemoryMap();
+}
+
+bool Bus::ReallocateMemoryMap(Error* error)
+{
+  // Need to back up RAM+BIOS.
+  DynamicHeapArray<u8> ram_backup;
+  DynamicHeapArray<u8> bios_backup;
+
+  if (System::IsValid())
+  {
+    CPU::CodeCache::InvalidateAllRAMBlocks();
+    UpdateFastmemViews(CPUFastmemMode::Disabled);
+
+    ram_backup.resize(RAM_8MB_SIZE);
+    std::memcpy(ram_backup.data(), g_unprotected_ram, RAM_8MB_SIZE);
+    bios_backup.resize(BIOS_SIZE);
+    std::memcpy(bios_backup.data(), g_bios, BIOS_SIZE);
+  }
+
+  ReleaseMemoryMap();
+  if (!AllocateMemoryMap(error)) [[unlikely]]
+    return false;
+
+  if (System::IsValid())
+  {
+    UpdateMappedRAMSize();
+    std::memcpy(g_unprotected_ram, ram_backup.data(), RAM_8MB_SIZE);
+    std::memcpy(g_bios, bios_backup.data(), BIOS_SIZE);
+    UpdateFastmemViews(g_settings.cpu_fastmem_mode);
   }
+
+  return true;
+}
+
+void Bus::CleanupMemoryMap()
+{
+#if !defined(_WIN32) && !defined(__ANDROID__)
+  // This is only needed on Linux.
+  if (!s_shmem_name.empty())
+    MemMap::DeleteSharedMemory(s_shmem_name.c_str());
+#endif
 }
 
 bool Bus::Initialize()
@@ -337,12 +402,6 @@ void Bus::Shutdown()
 
   g_ram_mask = 0;
   g_ram_size = 0;
-
-#ifndef __ANDROID__
-  Exports::RAM = 0;
-  Exports::RAM_SIZE = 0;
-  Exports::RAM_MASK = 0;
-#endif
 }
 
 void Bus::Reset()