Add modified o1Heap library and set up recomp heap

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "N64Recomp"]
 	path = N64Recomp
 	url = https://github.com/N64Recomp/N64Recomp
+[submodule "thirdparty/o1heap"]
+	path = thirdparty/o1heap
+	url = https://github.com/N64Recomp/o1heap

librecomp/CMakeLists.txt

@@ -14,6 +14,7 @@ add_library(librecomp STATIC
     "${CMAKE_CURRENT_SOURCE_DIR}/src/euc-jp.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/src/files.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/src/flash.cpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/heap.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/src/math_routines.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/src/mods.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/src/mod_events.cpp"
@@ -27,13 +28,16 @@ add_library(librecomp STATIC
     "${CMAKE_CURRENT_SOURCE_DIR}/src/sp.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/src/ultra_stubs.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/src/ultra_translation.cpp"
-    "${CMAKE_CURRENT_SOURCE_DIR}/src/vi.cpp")
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/vi.cpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/o1heap/o1heap/o1heap.c"
+)
 
 target_include_directories(librecomp PUBLIC
     "${CMAKE_CURRENT_SOURCE_DIR}/include"
     "${PROJECT_SOURCE_DIR}/../ultramodern/include"
     "${PROJECT_SOURCE_DIR}/../thirdparty"
     "${PROJECT_SOURCE_DIR}/../thirdparty/concurrentqueue"
+    "${PROJECT_SOURCE_DIR}/../thirdparty/o1heap"
 )
 target_include_directories(librecomp PRIVATE
     "${CMAKE_CURRENT_SOURCE_DIR}/include/librecomp"

librecomp/include/librecomp/addresses.hpp

@@ -3,8 +3,11 @@
 
 #include <cstdint>
 #include "ultramodern/ultra64.h"
+#include "librecomp/recomp.h"
 
 namespace recomp {
+    // 2GB (Addressable upper half of rdram)
+    constexpr size_t mem_size = 2U * 1024U * 1024U * 1024U;
     // We need a place in rdram to hold the PI handles, so pick an address in extended rdram
     constexpr int32_t cart_handle = 0x80800000;
     constexpr int32_t drive_handle = (int32_t)(cart_handle + sizeof(OSPiHandle));
@@ -20,6 +23,14 @@ namespace recomp {
     constexpr uint32_t sram_base = 0x08000000;
     constexpr uint32_t rom_base = 0x10000000;
     constexpr uint32_t drive_base = 0x06000000;
+
+    void register_heap_exports();
+    void init_heap(uint8_t* rdram, uint32_t address);
+    void* alloc(uint8_t* rdram, size_t size);
+    void free(uint8_t* rdram, void* mem);
 }
 
+extern "C" void recomp_alloc(uint8_t* rdram, recomp_context* ctx);
+extern "C" void recomp_free(uint8_t* rdram, recomp_context* ctx);
+
 #endif

librecomp/include/librecomp/overlays.hpp

@@ -23,6 +23,7 @@ namespace recomp {
         void register_overlays(const overlay_section_table_data_t& sections, const overlays_by_index_t& overlays);
 
         void register_patches(const char* patch_data, size_t patch_size, SectionTableEntry* code_sections, size_t num_sections);
+        void register_base_export(const std::string& name, recomp_func_t* func);
         void register_base_exports(const FunctionExport* exports);
         void register_base_events(char const* const* event_names);
         void read_patch_data(uint8_t* rdram, gpr patch_data_address);

librecomp/src/heap.cpp

@@ -0,0 +1,45 @@
+#include "o1heap/o1heap.h"
+
+#include "librecomp/addresses.hpp"
+#include "librecomp/overlays.hpp"
+
+static uint32_t heap_offset;
+
+static inline O1HeapInstance* get_heap(uint8_t* rdram) {
+    return reinterpret_cast<O1HeapInstance*>(&rdram[heap_offset]);
+}
+
+extern "C" void recomp_alloc(uint8_t* rdram, recomp_context* ctx) {
+    uint32_t offset = reinterpret_cast<uint8_t*>(recomp::alloc(rdram, ctx->r4)) - rdram;
+    ctx->r2 = offset + 0xFFFFFFFF80000000ULL;
+}
+
+extern "C" void recomp_free(uint8_t* rdram, recomp_context* ctx) {
+    recomp::free(rdram, TO_PTR(void, ctx->r4));
+}
+
+void recomp::register_heap_exports() {
+    recomp::overlays::register_base_export("recomp_alloc", recomp_alloc);
+    recomp::overlays::register_base_export("recomp_free", recomp_free);
+}
+
+void recomp::init_heap(uint8_t* rdram, uint32_t address) {
+    // Align the heap start to 16 bytes.
+    address = (address + 15U) & ~15U;
+
+    // Calculate the offset of the heap from the start of rdram and the size of the heap.
+    heap_offset = address - 0x80000000U;
+    size_t heap_size = recomp::mem_size - heap_offset;
+
+    printf("Initializing recomp heap at offset 0x%08X with size 0x%08X\n", heap_offset, static_cast<uint32_t>(heap_size));
+
+    o1heapInit(&rdram[heap_offset], heap_size);
+}
+
+void* recomp::alloc(uint8_t* rdram, size_t size) {
+    return o1heapAllocate(get_heap(rdram), size);
+}
+
+void recomp::free(uint8_t* rdram, void* mem) {
+    o1heapFree(get_heap(rdram), mem);
+}

librecomp/src/overlays.cpp

@@ -55,6 +55,10 @@ void recomp::overlays::register_patches(const char* patch, std::size_t size, Sec
     std::memcpy(patch_data.data(), patch, size);
 }
 
+void recomp::overlays::register_base_export(const std::string& name, recomp_func_t* func) {
+    base_exports.emplace(name, func);
+}
+
 void recomp::overlays::register_base_exports(const FunctionExport* export_list) {
     std::unordered_map<uint32_t, recomp_func_t*> patch_func_vram_map{};
 

librecomp/src/recomp.cpp

@@ -514,8 +514,8 @@ bool wait_for_game_started(uint8_t* rdram, recomp_context* context) {
 
                 init(rdram, context, game_entry.entrypoint_address);
 
+                uint32_t mod_ram_used = 0;
                 if (!game_entry.mod_game_id.empty()) {
-                    uint32_t mod_ram_used = 0;
                     std::vector<recomp::mods::ModLoadErrorDetails> mod_load_errors;
                     {
                         std::lock_guard lock { mod_context_mutex };
@@ -537,7 +537,9 @@ bool wait_for_game_started(uint8_t* rdram, recomp_context* context) {
                         return false;
                     }
                 }
-
+
+                recomp::init_heap(rdram, recomp::mod_rdram_start + mod_ram_used);
+
                 ultramodern::init_saving(rdram);
                 ultramodern::load_shader_cache(game_entry.cache_data);
 
@@ -599,9 +601,24 @@ void recomp::start(
         }
     }
 
-    // Allocate rdram_buffer
-    std::unique_ptr<uint8_t[]> rdram_buffer = std::make_unique<uint8_t[]>(rdram_size);
-    std::memset(rdram_buffer.get(), 0, rdram_size);
+    // Allocate rdram without comitting it. Use a platform-specific virtual allocation function
+    // that initializes to zero.
+    uint8_t* rdram;
+    bool alloc_failed;
+#ifdef _WIN32
+    rdram = reinterpret_cast<uint8_t*>(VirtualAlloc(nullptr, mem_size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE));
+    alloc_failed = (rdram == nullptr);
+#else
+    uint8_t* rdram = (uint8_t*)mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
+    alloc_failed = rdram == reinterpret_cast<uint8_t*>(MAP_FAILED);
+#endif
+
+    if (alloc_failed) {
+        ultramodern::error_handling::message_box("Failed to allocate memory!");
+        return;
+    }
+
+    recomp::register_heap_exports();
 
     std::thread game_thread{[](ultramodern::renderer::WindowHandle window_handle, uint8_t* rdram) {
         debug_printf("[Recomp] Starting\n");
@@ -614,7 +631,7 @@ void recomp::start(
 
         // Loop until the game starts.
         while (!wait_for_game_started(rdram, &context)) {}
-    }, window_handle, rdram_buffer.get()};
+    }, window_handle, rdram};
 
     while (!exited) {
         ultramodern::sleep_milliseconds(1);
@@ -627,4 +644,18 @@ void recomp::start(
     ultramodern::join_event_threads();
     ultramodern::join_thread_cleaner_thread();
     ultramodern::join_saving_thread();
+
+    // Free rdram.
+    bool free_failed;
+#ifdef _WIN32
+    // VirtualFree returns zero on failure.
+    free_failed = (VirtualFree(rdram, 0, MEM_RELEASE) == 0);
+#else
+    // munmap returns -1 on failure.
+    free_failed = (munmap(rdram, mem_size) == -1);
+#endif
+
+    if (free_failed) {
+        printf("Failed to free rdram\n");
+    }
 }