From 54a4ddcdcef3a224c07d67189b50127df29a6ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 21 Aug 2023 02:55:42 +0200 Subject: [PATCH 01/71] feat: add ModDownloader API declaration --- NorthstarDLL/CMakeLists.txt | 2 +- NorthstarDLL/mods/autodownload/moddownloader.h | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 NorthstarDLL/mods/autodownload/moddownloader.h diff --git a/NorthstarDLL/CMakeLists.txt b/NorthstarDLL/CMakeLists.txt index 126812e9b..8e38b657f 100644 --- a/NorthstarDLL/CMakeLists.txt +++ b/NorthstarDLL/CMakeLists.txt @@ -149,7 +149,7 @@ add_library(NorthstarDLL SHARED "dllmain.cpp" "dllmain.h" "ns_version.h" -) + "mods/autodownload/moddownloader.h") target_link_libraries(NorthstarDLL PRIVATE minhook diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h new file mode 100644 index 000000000..b57d29431 --- /dev/null +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -0,0 +1,18 @@ +class ModDownloader +{ + private: + rapidjson_document authorizedMods; + std::vector downloadingMods {}; + void FetchModsListFromAPI(); + + fs::path FetchModFromDistantStore(std::string modName); + bool IsModLegit(fs::path modPath, std::string expectedChecksum); + void ExtractMod(fs::path modPath); + + public: + ModDownloader(); + bool DownloadMod(std::string modName, std::string modVersion); + bool IsModAuthorized(std::string modName, std::string modVersion); + std::string GetModInstallProgress(std::string modName); // TODO return a struct + void CancelDownload(std::string modName); +}; From 0207d0d99b4d22cfbbcb53b33c23788f2bcdaf31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 21 Aug 2023 23:53:56 +0200 Subject: [PATCH 02/71] docs: add documentation to header --- .../mods/autodownload/moddownloader.h | 139 +++++++++++++++++- 1 file changed, 134 insertions(+), 5 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index b57d29431..9dd3b26c8 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -1,18 +1,147 @@ class ModDownloader { private: - rapidjson_document authorizedMods; - std::vector downloadingMods {}; + /** + * Retrieves the verified mods list from the central authority. + * + * The Northstar auto-downloading feature does NOT allow automatically installing + * all mods for various (notably security) reasons; mods that are candidate to + * auto-downloading are rather listed on a GitHub repository + * (https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json), + * which this method gets via a HTTP call to load into local state. + * + * If list fetching fails, local mods list will be initialized as empty, thus + * preventing any mod from being auto-downloaded. + * + * @returns nothing + */ void FetchModsListFromAPI(); - fs::path FetchModFromDistantStore(std::string modName); + struct VerifiedModVersion + { + std::string version; + std::string checksum; + }; + struct VerifiedModDetails + { + std::string dependencyPrefix; + std::vector versions; + }; + std::unordered_map verifiedMods = {}; + + /** + * Downloads a mod archive from distant store. + * + * This rebuilds the URI of the mod archive using both a predefined store URI + * and the mod dependency string from the `verifiedMods` variabl; fetched + * archive is then stored in a temporary location. + * + * @param modName name of the mod to be downloaded + * @param modVersion version of the mod to be downloaded + * @returns location of the downloaded archive + */ + fs::path FetchModFromDistantStore(std::string modName, std::string modVersion); + + /** + * Tells if a mod archive has not been corrupted. + * + * The mod validation procedure includes computing the SHA256 hash of the final + * archive, which is stored in the verified mods list. This hash is used by this + * very method to ensure the archive downloaded from the Internet is the exact + * same that has been manually verified. + * + * @param modPath path of the archive to check + * @param expectedChecksum checksum the archive should have + * @returns whether archive is legit + */ bool IsModLegit(fs::path modPath, std::string expectedChecksum); + + /** + * Extracts a mod archive to the game folder. + * + * This extracts a downloaded mod archive from its original location to the + * current game profile, in the remote mods folder. + * + * @param modPath location of the downloaded archive + * @returns nothing + */ void ExtractMod(fs::path modPath); public: ModDownloader(); - bool DownloadMod(std::string modName, std::string modVersion); + + /** + * Downloads a given mod from Thunderstore API to local game profile. + * + * @param modName name of the mod to be downloaded + * @param modVersion version of the mod to be downloaded + * @returns nothing + **/ + void DownloadMod(std::string modName, std::string modVersion); + + /** + * Checks whether a mod is verified. + * + * A mod is deemed verified/authorized through a manual validation process that is + * described here: https://github.com/R2Northstar/VerifiedMods; in practice, a mod + * is considered authorized if their name AND exact version appear in the + * `verifiedMods` variable. + * + * @param modName name of the mod to be checked + * @param modVersion version of the mod to be checked, must follow semantic versioning + * @returns whether the mod is authorized and can be auto-downloaded + */ bool IsModAuthorized(std::string modName, std::string modVersion); - std::string GetModInstallProgress(std::string modName); // TODO return a struct + + + enum ModInstallState + { + // Normal installation process + DOWNLOADING, + CHECKSUMING, + EXTRACTING, + DONE, // Everything went great, mod can be used in-game + + // Errors + FAILED, // Generic error message, should be avoided as much as possible + FAILED_READING_ARCHIVE, + FAILED_WRITING_TO_DISK, + MOD_FETCHING_FAILED, + MOD_CORRUPTED, // Downloaded archive checksum does not match verified hash + NO_DISK_SPACE_AVAILABLE, + NOT_FOUND // Mod is not currently being auto-downloaded + }; + + struct MOD_STATE + { + ModInstallState state; + int progress; + int total; + float ratio; + } modState; + + /** + * Retrieves installation information of a mod. + * + * Since mods are installed one at a time, mod installation data is stored in a global + * MOD_STATE instance. + * + * @param modName name of the mod to get information of + * @returns the class MOD_STATE instance + */ + MOD_STATE GetModInstallProgress(std::string modName); + + /** + * Cancels installation of a mod. + * + * Prevents installation of a mod currently being installed, no matter the install + * progress (downloading, checksuming, extracting), and frees all resources currently + * being used in this purpose. + * + * Does nothing if mod passed as argument is not being installed. + * + * @param modName name of the mod to prevent installation of + * @returns nothing + */ void CancelDownload(std::string modName); }; From 7a873571ecaa716d3337c109340756710a685a40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 21 Aug 2023 23:56:54 +0200 Subject: [PATCH 03/71] refactor: declare header as .hpp file --- NorthstarDLL/CMakeLists.txt | 2 +- .../mods/autodownload/{moddownloader.h => moddownloader.hpp} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename NorthstarDLL/mods/autodownload/{moddownloader.h => moddownloader.hpp} (100%) diff --git a/NorthstarDLL/CMakeLists.txt b/NorthstarDLL/CMakeLists.txt index 8e38b657f..8fde7d26c 100644 --- a/NorthstarDLL/CMakeLists.txt +++ b/NorthstarDLL/CMakeLists.txt @@ -149,7 +149,7 @@ add_library(NorthstarDLL SHARED "dllmain.cpp" "dllmain.h" "ns_version.h" - "mods/autodownload/moddownloader.h") + "mods/autodownload/moddownloader.hpp") target_link_libraries(NorthstarDLL PRIVATE minhook diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.hpp similarity index 100% rename from NorthstarDLL/mods/autodownload/moddownloader.h rename to NorthstarDLL/mods/autodownload/moddownloader.hpp From 8a40845411a748c8abd9842bada37ebd6232b2b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Tue, 22 Aug 2023 00:24:35 +0200 Subject: [PATCH 04/71] feat: add basic FetchModsListFromAPI implementation --- NorthstarDLL/CMakeLists.txt | 2 +- .../mods/autodownload/moddownloader.cpp | 20 ++++++++++++++++++ .../{moddownloader.hpp => moddownloader.h} | 21 +++++++++++-------- 3 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 NorthstarDLL/mods/autodownload/moddownloader.cpp rename NorthstarDLL/mods/autodownload/{moddownloader.hpp => moddownloader.h} (87%) diff --git a/NorthstarDLL/CMakeLists.txt b/NorthstarDLL/CMakeLists.txt index 8fde7d26c..a46f5a66a 100644 --- a/NorthstarDLL/CMakeLists.txt +++ b/NorthstarDLL/CMakeLists.txt @@ -149,7 +149,7 @@ add_library(NorthstarDLL SHARED "dllmain.cpp" "dllmain.h" "ns_version.h" - "mods/autodownload/moddownloader.hpp") + "mods/autodownload/moddownloader.hpp" "mods/autodownload/moddownloader.cpp") target_link_libraries(NorthstarDLL PRIVATE minhook diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp new file mode 100644 index 000000000..4a22c70c9 --- /dev/null +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -0,0 +1,20 @@ +#include "moddownloader.h" + +void ModDownloader::FetchModsListFromAPI() { + const char* url = MODS_LIST_URL; + + std::thread requestThread( + [url]() + { + curl_global_init(CURL_GLOBAL_ALL); + CURL* easyhandle = curl_easy_init(); + std::string readBuffer; + + curl_easy_setopt(easyhandle, CURLOPT_URL, url); + curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1L); + curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, &readBuffer); + + curl_easy_perform(easyhandle); + std::cout << readBuffer << std::endl; + }); +} diff --git a/NorthstarDLL/mods/autodownload/moddownloader.hpp b/NorthstarDLL/mods/autodownload/moddownloader.h similarity index 87% rename from NorthstarDLL/mods/autodownload/moddownloader.hpp rename to NorthstarDLL/mods/autodownload/moddownloader.h index 9dd3b26c8..e3b8c07fd 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.hpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -1,6 +1,9 @@ class ModDownloader { private: + const char* STORE_URL = "https://gcdn.thunderstore.io/live/repository/packages/"; + const char* MODS_LIST_URL = "https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json"; + /** * Retrieves the verified mods list from the central authority. * @@ -19,12 +22,12 @@ class ModDownloader struct VerifiedModVersion { - std::string version; - std::string checksum; + char* version; + char* checksum; }; struct VerifiedModDetails { - std::string dependencyPrefix; + char* dependencyPrefix; std::vector versions; }; std::unordered_map verifiedMods = {}; @@ -40,7 +43,7 @@ class ModDownloader * @param modVersion version of the mod to be downloaded * @returns location of the downloaded archive */ - fs::path FetchModFromDistantStore(std::string modName, std::string modVersion); + fs::path FetchModFromDistantStore(char* modName, char* modVersion); /** * Tells if a mod archive has not been corrupted. @@ -54,7 +57,7 @@ class ModDownloader * @param expectedChecksum checksum the archive should have * @returns whether archive is legit */ - bool IsModLegit(fs::path modPath, std::string expectedChecksum); + bool IsModLegit(fs::path modPath, char* expectedChecksum); /** * Extracts a mod archive to the game folder. @@ -77,7 +80,7 @@ class ModDownloader * @param modVersion version of the mod to be downloaded * @returns nothing **/ - void DownloadMod(std::string modName, std::string modVersion); + void DownloadMod(char* modName, char* modVersion); /** * Checks whether a mod is verified. @@ -91,7 +94,7 @@ class ModDownloader * @param modVersion version of the mod to be checked, must follow semantic versioning * @returns whether the mod is authorized and can be auto-downloaded */ - bool IsModAuthorized(std::string modName, std::string modVersion); + bool IsModAuthorized(char* modName, char* modVersion); enum ModInstallState @@ -129,7 +132,7 @@ class ModDownloader * @param modName name of the mod to get information of * @returns the class MOD_STATE instance */ - MOD_STATE GetModInstallProgress(std::string modName); + MOD_STATE GetModInstallProgress(char* modName); /** * Cancels installation of a mod. @@ -143,5 +146,5 @@ class ModDownloader * @param modName name of the mod to prevent installation of * @returns nothing */ - void CancelDownload(std::string modName); + void CancelDownload(char* modName); }; From e5f84eee5fdec17b4c7482dd8f7aaa77edd5cadc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Tue, 22 Aug 2023 00:30:08 +0200 Subject: [PATCH 05/71] feat: add global ModDownloader instance --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 4a22c70c9..4cd11a908 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -1,5 +1,7 @@ #include "moddownloader.h" +ModDownloader* g_pModDownloader; + void ModDownloader::FetchModsListFromAPI() { const char* url = MODS_LIST_URL; @@ -18,3 +20,8 @@ void ModDownloader::FetchModsListFromAPI() { std::cout << readBuffer << std::endl; }); } + +ON_DLL_LOAD_RELIESON("engine.dll", ModDownloader, (ConCommand, MasterServer), (CModule module)) +{ + g_pModDownloader = new ModDownloader; +} From 8a16ec57a3363e445851814a9beab5efc1fd2446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Tue, 22 Aug 2023 00:52:17 +0200 Subject: [PATCH 06/71] feat: add command to call FetchModsListFromAPI from in-game console --- .../mods/autodownload/moddownloader.cpp | 15 +++++++-- .../mods/autodownload/moddownloader.h | 32 +++++++++---------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 4cd11a908..bf11d4826 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -2,7 +2,10 @@ ModDownloader* g_pModDownloader; -void ModDownloader::FetchModsListFromAPI() { +ModDownloader::ModDownloader() {} + +void ModDownloader::FetchModsListFromAPI() +{ const char* url = MODS_LIST_URL; std::thread requestThread( @@ -21,7 +24,13 @@ void ModDownloader::FetchModsListFromAPI() { }); } -ON_DLL_LOAD_RELIESON("engine.dll", ModDownloader, (ConCommand, MasterServer), (CModule module)) +void ConCommand_fetch_verified_mods(const CCommand& args) +{ + g_pModDownloader->FetchModsListFromAPI(); +} + +ON_DLL_LOAD_RELIESON("engine.dll", ModDownloader, (ConCommand), (CModule module)) { - g_pModDownloader = new ModDownloader; + g_pModDownloader = new ModDownloader(); + RegisterConCommand("fetch_verified_mods", ConCommand_fetch_verified_mods, "fetches verified mods list from GitHub repository", FCVAR_NONE); } diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index e3b8c07fd..b5e3c596b 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -4,22 +4,6 @@ class ModDownloader const char* STORE_URL = "https://gcdn.thunderstore.io/live/repository/packages/"; const char* MODS_LIST_URL = "https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json"; - /** - * Retrieves the verified mods list from the central authority. - * - * The Northstar auto-downloading feature does NOT allow automatically installing - * all mods for various (notably security) reasons; mods that are candidate to - * auto-downloading are rather listed on a GitHub repository - * (https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json), - * which this method gets via a HTTP call to load into local state. - * - * If list fetching fails, local mods list will be initialized as empty, thus - * preventing any mod from being auto-downloaded. - * - * @returns nothing - */ - void FetchModsListFromAPI(); - struct VerifiedModVersion { char* version; @@ -73,6 +57,22 @@ class ModDownloader public: ModDownloader(); + /** + * Retrieves the verified mods list from the central authority. + * + * The Northstar auto-downloading feature does NOT allow automatically installing + * all mods for various (notably security) reasons; mods that are candidate to + * auto-downloading are rather listed on a GitHub repository + * (https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json), + * which this method gets via a HTTP call to load into local state. + * + * If list fetching fails, local mods list will be initialized as empty, thus + * preventing any mod from being auto-downloaded. + * + * @returns nothing + */ + void FetchModsListFromAPI(); + /** * Downloads a given mod from Thunderstore API to local game profile. * From f4e0a99d8b5f5308413808d011f225d5f5324744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Tue, 22 Aug 2023 01:48:16 +0200 Subject: [PATCH 07/71] fix: adjust header file extension --- NorthstarDLL/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NorthstarDLL/CMakeLists.txt b/NorthstarDLL/CMakeLists.txt index a46f5a66a..602a6da70 100644 --- a/NorthstarDLL/CMakeLists.txt +++ b/NorthstarDLL/CMakeLists.txt @@ -149,7 +149,7 @@ add_library(NorthstarDLL SHARED "dllmain.cpp" "dllmain.h" "ns_version.h" - "mods/autodownload/moddownloader.hpp" "mods/autodownload/moddownloader.cpp") + "mods/autodownload/moddownloader.h" "mods/autodownload/moddownloader.cpp") target_link_libraries(NorthstarDLL PRIVATE minhook From 053fa83f98dfd0b0bf52df6d74ac047944c384e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 01:07:17 +0200 Subject: [PATCH 08/71] feat: complete FetchModsListFromAPI implementation --- .../mods/autodownload/moddownloader.cpp | 80 ++++++++++++++++--- 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index bf11d4826..252e09e6f 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -1,27 +1,83 @@ #include "moddownloader.h" +#include ModDownloader* g_pModDownloader; -ModDownloader::ModDownloader() {} +ModDownloader::ModDownloader() { + spdlog::info("Mod downloaded initialized"); + modState = {}; +} -void ModDownloader::FetchModsListFromAPI() +size_t write_to_string(void* ptr, size_t size, size_t count, void* stream) { - const char* url = MODS_LIST_URL; + ((std::string*)stream)->append((char*)ptr, 0, size * count); + return size * count; +} +void ModDownloader::FetchModsListFromAPI() +{ std::thread requestThread( - [url]() + [this]() { - curl_global_init(CURL_GLOBAL_ALL); - CURL* easyhandle = curl_easy_init(); - std::string readBuffer; + CURLcode result; + CURL* easyhandle; + rapidjson::Document verifiedModsJson; + std::string url = MODS_LIST_URL; + + curl_global_init(CURL_GLOBAL_ALL); + easyhandle = curl_easy_init(); + std::string readBuffer; + + // Fetching mods list from GitHub repository + curl_easy_setopt(easyhandle, CURLOPT_CUSTOMREQUEST, "GET"); + curl_easy_setopt(easyhandle, CURLOPT_TIMEOUT, 30L); + curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str()); + curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1L); + curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, &readBuffer); + curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, write_to_string); + result = curl_easy_perform(easyhandle); + + if (result == CURLcode::CURLE_OK) + { + spdlog::info("Mods list successfully fetched."); + } + else + { + spdlog::error("Fetching mods list failed: {}", curl_easy_strerror(result)); + goto REQUEST_END_CLEANUP; + } + + // Load mods list into local state + spdlog::info("Loading mods configuration..."); + verifiedModsJson.Parse(readBuffer); + for (auto i = verifiedModsJson.MemberBegin(); i != verifiedModsJson.MemberEnd(); ++i) + { + std::string name = i->name.GetString(); + std::string dependency = i->value["DependencyPrefix"].GetString(); + + std::vector modVersions; + rapidjson::Value& versions = i->value["Versions"]; + assert(versions.IsArray()); + for (rapidjson::Value::ConstValueIterator itr = versions.Begin(); itr != versions.End(); ++itr) + { + const rapidjson::Value& attribute = *itr; + assert(attribute.IsObject()); + std::string version = attribute["Version"].GetString(); + std::string checksum = attribute["Checksum"].GetString(); + modVersions.push_back({.version = (char*)version.c_str(), .checksum = (char*)checksum.c_str()}); + } + + VerifiedModDetails modConfig = {.dependencyPrefix = (char*)dependency.c_str(), .versions = modVersions}; + verifiedMods.insert({name, modConfig}); + spdlog::info("==> Loaded configuration for mod \"" + name + "\""); + } - curl_easy_setopt(easyhandle, CURLOPT_URL, url); - curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1L); - curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, &readBuffer); + spdlog::info("Done loading verified mods list."); - curl_easy_perform(easyhandle); - std::cout << readBuffer << std::endl; + REQUEST_END_CLEANUP: + curl_easy_cleanup(easyhandle); }); + requestThread.detach(); } void ConCommand_fetch_verified_mods(const CCommand& args) From 35afd31b7c67c3d96490e32b0751a2d9a9342ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 01:08:31 +0200 Subject: [PATCH 09/71] style: format --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 252e09e6f..db1f53232 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -3,7 +3,8 @@ ModDownloader* g_pModDownloader; -ModDownloader::ModDownloader() { +ModDownloader::ModDownloader() +{ spdlog::info("Mod downloaded initialized"); modState = {}; } @@ -72,11 +73,11 @@ void ModDownloader::FetchModsListFromAPI() spdlog::info("==> Loaded configuration for mod \"" + name + "\""); } - spdlog::info("Done loading verified mods list."); + spdlog::info("Done loading verified mods list."); REQUEST_END_CLEANUP: curl_easy_cleanup(easyhandle); - }); + }); requestThread.detach(); } @@ -88,5 +89,6 @@ void ConCommand_fetch_verified_mods(const CCommand& args) ON_DLL_LOAD_RELIESON("engine.dll", ModDownloader, (ConCommand), (CModule module)) { g_pModDownloader = new ModDownloader(); - RegisterConCommand("fetch_verified_mods", ConCommand_fetch_verified_mods, "fetches verified mods list from GitHub repository", FCVAR_NONE); + RegisterConCommand( + "fetch_verified_mods", ConCommand_fetch_verified_mods, "fetches verified mods list from GitHub repository", FCVAR_NONE); } From 180b432488db04eebc2a4161cee73843471d43b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 01:11:43 +0200 Subject: [PATCH 10/71] style: format --- .../mods/autodownload/moddownloader.h | 267 +++++++++--------- 1 file changed, 133 insertions(+), 134 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index b5e3c596b..4d76c80ea 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -1,150 +1,149 @@ class ModDownloader { - private: - const char* STORE_URL = "https://gcdn.thunderstore.io/live/repository/packages/"; - const char* MODS_LIST_URL = "https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json"; + private: + const char* STORE_URL = "https://gcdn.thunderstore.io/live/repository/packages/"; + const char* MODS_LIST_URL = "https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json"; - struct VerifiedModVersion - { - char* version; - char* checksum; - }; - struct VerifiedModDetails - { - char* dependencyPrefix; - std::vector versions; - }; - std::unordered_map verifiedMods = {}; - - /** - * Downloads a mod archive from distant store. - * - * This rebuilds the URI of the mod archive using both a predefined store URI - * and the mod dependency string from the `verifiedMods` variabl; fetched - * archive is then stored in a temporary location. - * - * @param modName name of the mod to be downloaded - * @param modVersion version of the mod to be downloaded - * @returns location of the downloaded archive - */ - fs::path FetchModFromDistantStore(char* modName, char* modVersion); + struct VerifiedModVersion + { + char* version; + char* checksum; + }; + struct VerifiedModDetails + { + char* dependencyPrefix; + std::vector versions; + }; + std::unordered_map verifiedMods = {}; - /** - * Tells if a mod archive has not been corrupted. - * - * The mod validation procedure includes computing the SHA256 hash of the final - * archive, which is stored in the verified mods list. This hash is used by this - * very method to ensure the archive downloaded from the Internet is the exact - * same that has been manually verified. - * - * @param modPath path of the archive to check - * @param expectedChecksum checksum the archive should have - * @returns whether archive is legit - */ - bool IsModLegit(fs::path modPath, char* expectedChecksum); + /** + * Downloads a mod archive from distant store. + * + * This rebuilds the URI of the mod archive using both a predefined store URI + * and the mod dependency string from the `verifiedMods` variabl; fetched + * archive is then stored in a temporary location. + * + * @param modName name of the mod to be downloaded + * @param modVersion version of the mod to be downloaded + * @returns location of the downloaded archive + */ + fs::path FetchModFromDistantStore(char* modName, char* modVersion); - /** - * Extracts a mod archive to the game folder. - * - * This extracts a downloaded mod archive from its original location to the - * current game profile, in the remote mods folder. - * - * @param modPath location of the downloaded archive - * @returns nothing - */ - void ExtractMod(fs::path modPath); + /** + * Tells if a mod archive has not been corrupted. + * + * The mod validation procedure includes computing the SHA256 hash of the final + * archive, which is stored in the verified mods list. This hash is used by this + * very method to ensure the archive downloaded from the Internet is the exact + * same that has been manually verified. + * + * @param modPath path of the archive to check + * @param expectedChecksum checksum the archive should have + * @returns whether archive is legit + */ + bool IsModLegit(fs::path modPath, char* expectedChecksum); - public: - ModDownloader(); + /** + * Extracts a mod archive to the game folder. + * + * This extracts a downloaded mod archive from its original location to the + * current game profile, in the remote mods folder. + * + * @param modPath location of the downloaded archive + * @returns nothing + */ + void ExtractMod(fs::path modPath); - /** - * Retrieves the verified mods list from the central authority. - * - * The Northstar auto-downloading feature does NOT allow automatically installing - * all mods for various (notably security) reasons; mods that are candidate to - * auto-downloading are rather listed on a GitHub repository - * (https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json), - * which this method gets via a HTTP call to load into local state. - * - * If list fetching fails, local mods list will be initialized as empty, thus - * preventing any mod from being auto-downloaded. - * - * @returns nothing - */ - void FetchModsListFromAPI(); + public: + ModDownloader(); - /** - * Downloads a given mod from Thunderstore API to local game profile. - * - * @param modName name of the mod to be downloaded - * @param modVersion version of the mod to be downloaded - * @returns nothing - **/ - void DownloadMod(char* modName, char* modVersion); + /** + * Retrieves the verified mods list from the central authority. + * + * The Northstar auto-downloading feature does NOT allow automatically installing + * all mods for various (notably security) reasons; mods that are candidate to + * auto-downloading are rather listed on a GitHub repository + * (https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json), + * which this method gets via a HTTP call to load into local state. + * + * If list fetching fails, local mods list will be initialized as empty, thus + * preventing any mod from being auto-downloaded. + * + * @returns nothing + */ + void FetchModsListFromAPI(); - /** - * Checks whether a mod is verified. - * - * A mod is deemed verified/authorized through a manual validation process that is - * described here: https://github.com/R2Northstar/VerifiedMods; in practice, a mod - * is considered authorized if their name AND exact version appear in the - * `verifiedMods` variable. - * - * @param modName name of the mod to be checked - * @param modVersion version of the mod to be checked, must follow semantic versioning - * @returns whether the mod is authorized and can be auto-downloaded - */ - bool IsModAuthorized(char* modName, char* modVersion); + /** + * Downloads a given mod from Thunderstore API to local game profile. + * + * @param modName name of the mod to be downloaded + * @param modVersion version of the mod to be downloaded + * @returns nothing + **/ + void DownloadMod(char* modName, char* modVersion); + /** + * Checks whether a mod is verified. + * + * A mod is deemed verified/authorized through a manual validation process that is + * described here: https://github.com/R2Northstar/VerifiedMods; in practice, a mod + * is considered authorized if their name AND exact version appear in the + * `verifiedMods` variable. + * + * @param modName name of the mod to be checked + * @param modVersion version of the mod to be checked, must follow semantic versioning + * @returns whether the mod is authorized and can be auto-downloaded + */ + bool IsModAuthorized(char* modName, char* modVersion); - enum ModInstallState - { - // Normal installation process - DOWNLOADING, - CHECKSUMING, - EXTRACTING, - DONE, // Everything went great, mod can be used in-game + enum ModInstallState + { + // Normal installation process + DOWNLOADING, + CHECKSUMING, + EXTRACTING, + DONE, // Everything went great, mod can be used in-game - // Errors - FAILED, // Generic error message, should be avoided as much as possible - FAILED_READING_ARCHIVE, - FAILED_WRITING_TO_DISK, - MOD_FETCHING_FAILED, - MOD_CORRUPTED, // Downloaded archive checksum does not match verified hash - NO_DISK_SPACE_AVAILABLE, - NOT_FOUND // Mod is not currently being auto-downloaded - }; + // Errors + FAILED, // Generic error message, should be avoided as much as possible + FAILED_READING_ARCHIVE, + FAILED_WRITING_TO_DISK, + MOD_FETCHING_FAILED, + MOD_CORRUPTED, // Downloaded archive checksum does not match verified hash + NO_DISK_SPACE_AVAILABLE, + NOT_FOUND // Mod is not currently being auto-downloaded + }; - struct MOD_STATE - { - ModInstallState state; - int progress; - int total; - float ratio; - } modState; + struct MOD_STATE + { + ModInstallState state; + int progress; + int total; + float ratio; + } modState; - /** - * Retrieves installation information of a mod. - * - * Since mods are installed one at a time, mod installation data is stored in a global - * MOD_STATE instance. - * - * @param modName name of the mod to get information of - * @returns the class MOD_STATE instance - */ - MOD_STATE GetModInstallProgress(char* modName); + /** + * Retrieves installation information of a mod. + * + * Since mods are installed one at a time, mod installation data is stored in a global + * MOD_STATE instance. + * + * @param modName name of the mod to get information of + * @returns the class MOD_STATE instance + */ + MOD_STATE GetModInstallProgress(char* modName); - /** - * Cancels installation of a mod. - * - * Prevents installation of a mod currently being installed, no matter the install - * progress (downloading, checksuming, extracting), and frees all resources currently - * being used in this purpose. - * - * Does nothing if mod passed as argument is not being installed. - * - * @param modName name of the mod to prevent installation of - * @returns nothing - */ - void CancelDownload(char* modName); + /** + * Cancels installation of a mod. + * + * Prevents installation of a mod currently being installed, no matter the install + * progress (downloading, checksuming, extracting), and frees all resources currently + * being used in this purpose. + * + * Does nothing if mod passed as argument is not being installed. + * + * @param modName name of the mod to prevent installation of + * @returns nothing + */ + void CancelDownload(char* modName); }; From 5dfda30700b281fa4c7ac3ea2e7893d5c9accb85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 11:32:08 +0200 Subject: [PATCH 11/71] feat: add IsModAuthorized implementation --- .../mods/autodownload/moddownloader.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index db1f53232..40afea888 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -81,6 +81,24 @@ void ModDownloader::FetchModsListFromAPI() requestThread.detach(); } +bool ModDownloader::IsModAuthorized(char* modName, char* modVersion) +{ + if (!verifiedMods.contains(modName)) + { + return false; + } + + std::vector versions = verifiedMods[modName].versions; + std::vector matchingVersions; + std::copy_if( + versions.begin(), + versions.end(), + std::back_inserter(matchingVersions), + [modVersion](VerifiedModVersion v) { return strcmp(modVersion, v.version) == 0; }); + + return matchingVersions.size() != 0; +} + void ConCommand_fetch_verified_mods(const CCommand& args) { g_pModDownloader->FetchModsListFromAPI(); From b4584e7bdd33d02182058d1bc65134285d3c4a78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 12:04:54 +0200 Subject: [PATCH 12/71] feat: add a ConCommand to invoke IsModAuthorized --- .../mods/autodownload/moddownloader.cpp | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 40afea888..3addcf9a5 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -104,9 +104,32 @@ void ConCommand_fetch_verified_mods(const CCommand& args) g_pModDownloader->FetchModsListFromAPI(); } +void ConCommand_is_mod_verified(const CCommand& args) +{ + if (args.ArgC() < 3) + { + return; + } + + // Split arguments string by whitespaces (https://stackoverflow.com/a/5208977) + std::string buf; + std::stringstream ss(args.ArgS()); + std::vector tokens; + while (ss >> buf) + tokens.push_back(buf); + + char* modName = (char*)tokens[0].c_str(); + char* modVersion = (char*)tokens[1].c_str(); + bool result = g_pModDownloader->IsModAuthorized(modName, modVersion); + std::string msg = std::format("Mod \"{}\" (version {}) is verified: {}", modName, modVersion, result); + spdlog::info(msg); +} + + ON_DLL_LOAD_RELIESON("engine.dll", ModDownloader, (ConCommand), (CModule module)) { g_pModDownloader = new ModDownloader(); RegisterConCommand( "fetch_verified_mods", ConCommand_fetch_verified_mods, "fetches verified mods list from GitHub repository", FCVAR_NONE); + RegisterConCommand("is_mod_verified", ConCommand_is_mod_verified, "checks if a mod is included in verified mods list", FCVAR_NONE); } From 61f47dd8fb246e63df1f4444323095d6f5644139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 14:50:54 +0200 Subject: [PATCH 13/71] fix: use std::string instead of char* in VerifiedModVersion struct --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 4 ++-- NorthstarDLL/mods/autodownload/moddownloader.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 3addcf9a5..a41cb9b26 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -65,7 +65,7 @@ void ModDownloader::FetchModsListFromAPI() assert(attribute.IsObject()); std::string version = attribute["Version"].GetString(); std::string checksum = attribute["Checksum"].GetString(); - modVersions.push_back({.version = (char*)version.c_str(), .checksum = (char*)checksum.c_str()}); + modVersions.push_back({.version = version, .checksum = checksum}); } VerifiedModDetails modConfig = {.dependencyPrefix = (char*)dependency.c_str(), .versions = modVersions}; @@ -94,7 +94,7 @@ bool ModDownloader::IsModAuthorized(char* modName, char* modVersion) versions.begin(), versions.end(), std::back_inserter(matchingVersions), - [modVersion](VerifiedModVersion v) { return strcmp(modVersion, v.version) == 0; }); + [modVersion](VerifiedModVersion v) { return strcmp(modVersion, v.version.c_str()) == 0; }); return matchingVersions.size() != 0; } diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index 4d76c80ea..b4c866cfd 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -6,8 +6,8 @@ class ModDownloader struct VerifiedModVersion { - char* version; - char* checksum; + std::string version; + std::string checksum; }; struct VerifiedModDetails { From f7e9dd809ec9892f670f350c823f6be2167e7df6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 15:13:40 +0200 Subject: [PATCH 14/71] feat: add DownloadMod skeleton --- .../mods/autodownload/moddownloader.cpp | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index a41cb9b26..e336ac86e 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -99,6 +99,29 @@ bool ModDownloader::IsModAuthorized(char* modName, char* modVersion) return matchingVersions.size() != 0; } +void ModDownloader::DownloadMod(char* modName, char* modVersion) +{ + // Check if mod can be auto-downloaded + if (!IsModAuthorized(modName, modVersion)) + { + spdlog::warn("Tried to download a mod that is not verified, aborting."); + return; + } + + // Download mod archive + std::string expectedHash = "TODO"; + fs::path archiveLocation = FetchModFromDistantStore(modName, modVersion); + if (!IsModLegit(archiveLocation, (char*)expectedHash.c_str())) + { + spdlog::info("Archive hash does not match expected checksum, aborting."); + return; + } + + // TODO extract mod archive +} + + + void ConCommand_fetch_verified_mods(const CCommand& args) { g_pModDownloader->FetchModsListFromAPI(); From d30fa9fd62895dbef2a75d98c7ed8939864e8b53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 15:44:15 +0200 Subject: [PATCH 15/71] feat: add a ConCommand to invoke DownloadMod --- .../mods/autodownload/moddownloader.cpp | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index e336ac86e..2903e3021 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -81,6 +81,16 @@ void ModDownloader::FetchModsListFromAPI() requestThread.detach(); } +fs::path ModDownloader::FetchModFromDistantStore(char* modName, char* modVersion) +{ + return fs::path(); +} + +bool ModDownloader::IsModLegit(fs::path modPath, char* expectedChecksum) +{ + return false; +} + bool ModDownloader::IsModAuthorized(char* modName, char* modVersion) { if (!verifiedMods.contains(modName)) @@ -113,7 +123,7 @@ void ModDownloader::DownloadMod(char* modName, char* modVersion) fs::path archiveLocation = FetchModFromDistantStore(modName, modVersion); if (!IsModLegit(archiveLocation, (char*)expectedHash.c_str())) { - spdlog::info("Archive hash does not match expected checksum, aborting."); + spdlog::warn("Archive hash does not match expected checksum, aborting."); return; } @@ -148,6 +158,25 @@ void ConCommand_is_mod_verified(const CCommand& args) spdlog::info(msg); } +void ConCommand_download_mod(const CCommand& args) +{ + if (args.ArgC() < 3) + { + return; + } + + // Split arguments string by whitespaces (https://stackoverflow.com/a/5208977) + std::string buf; + std::stringstream ss(args.ArgS()); + std::vector tokens; + while (ss >> buf) + tokens.push_back(buf); + + char* modName = (char*)tokens[0].c_str(); + char* modVersion = (char*)tokens[1].c_str(); + g_pModDownloader->DownloadMod(modName, modVersion); +} + ON_DLL_LOAD_RELIESON("engine.dll", ModDownloader, (ConCommand), (CModule module)) { @@ -155,4 +184,5 @@ ON_DLL_LOAD_RELIESON("engine.dll", ModDownloader, (ConCommand), (CModule module) RegisterConCommand( "fetch_verified_mods", ConCommand_fetch_verified_mods, "fetches verified mods list from GitHub repository", FCVAR_NONE); RegisterConCommand("is_mod_verified", ConCommand_is_mod_verified, "checks if a mod is included in verified mods list", FCVAR_NONE); + RegisterConCommand("download_mod", ConCommand_download_mod, "downloads a mod from remote store", FCVAR_NONE); } From 2dde90658c65db14e8e22eccca53331c5c6a72c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 15:45:37 +0200 Subject: [PATCH 16/71] fix: use std::string instead of char* in VerifiedModDetails struct --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 2 +- NorthstarDLL/mods/autodownload/moddownloader.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 2903e3021..6dac7a3ef 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -68,7 +68,7 @@ void ModDownloader::FetchModsListFromAPI() modVersions.push_back({.version = version, .checksum = checksum}); } - VerifiedModDetails modConfig = {.dependencyPrefix = (char*)dependency.c_str(), .versions = modVersions}; + VerifiedModDetails modConfig = {.dependencyPrefix = dependency, .versions = modVersions}; verifiedMods.insert({name, modConfig}); spdlog::info("==> Loaded configuration for mod \"" + name + "\""); } diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index b4c866cfd..0dad57e90 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -11,7 +11,7 @@ class ModDownloader }; struct VerifiedModDetails { - char* dependencyPrefix; + std::string dependencyPrefix; std::vector versions; }; std::unordered_map verifiedMods = {}; From 9b041d70b58592e0ed688e6f7749eee5e93fa20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 16:08:37 +0200 Subject: [PATCH 17/71] feat: add FetchModFromDistantStore implementation --- .../mods/autodownload/moddownloader.cpp | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 6dac7a3ef..b83cda5eb 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -81,9 +81,56 @@ void ModDownloader::FetchModsListFromAPI() requestThread.detach(); } +size_t write_data(void* ptr, size_t size, size_t nmemb, FILE* stream) +{ + size_t written; + written = fwrite(ptr, size, nmemb, stream); + return written; +} + fs::path ModDownloader::FetchModFromDistantStore(char* modName, char* modVersion) { - return fs::path(); + // Build archive distant URI + std::string archiveName = std::format("{}-{}.zip", verifiedMods[modName].dependencyPrefix, modVersion); + std::string url = STORE_URL + archiveName; + spdlog::info(std::format("Fetching mod archive from {}", url)); + + // Download destination + std::filesystem::path downloadPath = std::filesystem::temp_directory_path() / archiveName; + + // Download the actual archive + std::thread requestThread( + [this, url, downloadPath]() + { + FILE* fp = fopen(downloadPath.generic_string().c_str(), "wb"); + CURLcode result; + CURL* easyhandle; + easyhandle = curl_easy_init(); + + curl_easy_setopt(easyhandle, CURLOPT_TIMEOUT, 30L); + curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str()); + curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1L); + curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, write_data); + result = curl_easy_perform(easyhandle); + + if (result == CURLcode::CURLE_OK) + { + spdlog::info("Mod archive successfully fetched."); + } + else + { + spdlog::error("Fetching mod archive failed: {}", curl_easy_strerror(result)); + goto REQUEST_END_CLEANUP; + } + + REQUEST_END_CLEANUP: + curl_easy_cleanup(easyhandle); + fclose(fp); + }); + + requestThread.join(); + return downloadPath; } bool ModDownloader::IsModLegit(fs::path modPath, char* expectedChecksum) From 5a1a54ca96eb61cc60266775200f2cdb8743d922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 16:17:01 +0200 Subject: [PATCH 18/71] fix: DownloadMod uses a thread to avoid UI freezing --- .../mods/autodownload/moddownloader.cpp | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index b83cda5eb..418686d92 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -165,16 +165,27 @@ void ModDownloader::DownloadMod(char* modName, char* modVersion) return; } - // Download mod archive - std::string expectedHash = "TODO"; - fs::path archiveLocation = FetchModFromDistantStore(modName, modVersion); - if (!IsModLegit(archiveLocation, (char*)expectedHash.c_str())) - { - spdlog::warn("Archive hash does not match expected checksum, aborting."); - return; - } + std::thread requestThread( + [this, modName, modVersion]() + { + // Download mod archive + std::string expectedHash = "TODO"; + fs::path archiveLocation = FetchModFromDistantStore(modName, modVersion); + if (!IsModLegit(archiveLocation, (char*)expectedHash.c_str())) + { + spdlog::warn("Archive hash does not match expected checksum, aborting."); + return; + } - // TODO extract mod archive + spdlog::info("Hash OK"); + + // TODO extract mod archive + + REQUEST_END_CLEANUP: + spdlog::info("ok"); + }); + + requestThread.detach(); } From c4af53d7054d8dc0a1e88fe8c90fd241a4fba0b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 16:34:43 +0200 Subject: [PATCH 19/71] refactor: use a map to store mod version information to ease their retrieval --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 15 ++++----------- NorthstarDLL/mods/autodownload/moddownloader.h | 3 +-- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 418686d92..7bfce4be0 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -56,7 +56,7 @@ void ModDownloader::FetchModsListFromAPI() std::string name = i->name.GetString(); std::string dependency = i->value["DependencyPrefix"].GetString(); - std::vector modVersions; + std::unordered_map modVersions; rapidjson::Value& versions = i->value["Versions"]; assert(versions.IsArray()); for (rapidjson::Value::ConstValueIterator itr = versions.Begin(); itr != versions.End(); ++itr) @@ -65,7 +65,7 @@ void ModDownloader::FetchModsListFromAPI() assert(attribute.IsObject()); std::string version = attribute["Version"].GetString(); std::string checksum = attribute["Checksum"].GetString(); - modVersions.push_back({.version = version, .checksum = checksum}); + modVersions.insert({version, {.checksum = checksum}}); } VerifiedModDetails modConfig = {.dependencyPrefix = dependency, .versions = modVersions}; @@ -145,15 +145,8 @@ bool ModDownloader::IsModAuthorized(char* modName, char* modVersion) return false; } - std::vector versions = verifiedMods[modName].versions; - std::vector matchingVersions; - std::copy_if( - versions.begin(), - versions.end(), - std::back_inserter(matchingVersions), - [modVersion](VerifiedModVersion v) { return strcmp(modVersion, v.version.c_str()) == 0; }); - - return matchingVersions.size() != 0; + std::unordered_map versions = verifiedMods[modName].versions; + return versions.count(modVersion) != 0; } void ModDownloader::DownloadMod(char* modName, char* modVersion) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index 0dad57e90..fba053d25 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -6,13 +6,12 @@ class ModDownloader struct VerifiedModVersion { - std::string version; std::string checksum; }; struct VerifiedModDetails { std::string dependencyPrefix; - std::vector versions; + std::unordered_map versions = {}; }; std::unordered_map verifiedMods = {}; From 7ed2f4301507ab7c8a1e33b502ac7b372843ec88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 16:51:12 +0200 Subject: [PATCH 20/71] feat: bypass checksum verification for now --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 7bfce4be0..47f6e7c99 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -135,7 +135,8 @@ fs::path ModDownloader::FetchModFromDistantStore(char* modName, char* modVersion bool ModDownloader::IsModLegit(fs::path modPath, char* expectedChecksum) { - return false; + // TODO implement + return true; } bool ModDownloader::IsModAuthorized(char* modName, char* modVersion) @@ -162,7 +163,7 @@ void ModDownloader::DownloadMod(char* modName, char* modVersion) [this, modName, modVersion]() { // Download mod archive - std::string expectedHash = "TODO"; + std::string expectedHash = verifiedMods[modName].versions[modVersion].checksum; fs::path archiveLocation = FetchModFromDistantStore(modName, modVersion); if (!IsModLegit(archiveLocation, (char*)expectedHash.c_str())) { @@ -170,8 +171,6 @@ void ModDownloader::DownloadMod(char* modName, char* modVersion) return; } - spdlog::info("Hash OK"); - // TODO extract mod archive REQUEST_END_CLEANUP: From 29a735145c0ccfae0d87398e67bd5237df13ac2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 23 Aug 2023 16:52:02 +0200 Subject: [PATCH 21/71] style: apply clang formatting --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 47f6e7c99..0bcd95f82 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -180,8 +180,6 @@ void ModDownloader::DownloadMod(char* modName, char* modVersion) requestThread.detach(); } - - void ConCommand_fetch_verified_mods(const CCommand& args) { g_pModDownloader->FetchModsListFromAPI(); @@ -227,7 +225,6 @@ void ConCommand_download_mod(const CCommand& args) g_pModDownloader->DownloadMod(modName, modVersion); } - ON_DLL_LOAD_RELIESON("engine.dll", ModDownloader, (ConCommand), (CModule module)) { g_pModDownloader = new ModDownloader(); From 840771d5e11dc67bdada47a764668d6bf813027e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Thu, 24 Aug 2023 16:31:59 +0200 Subject: [PATCH 22/71] docs: add archive download destination log --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 0bcd95f82..54018cc07 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -97,6 +97,7 @@ fs::path ModDownloader::FetchModFromDistantStore(char* modName, char* modVersion // Download destination std::filesystem::path downloadPath = std::filesystem::temp_directory_path() / archiveName; + spdlog::info(std::format("Downloading archive to {}", downloadPath.generic_string())); // Download the actual archive std::thread requestThread( From edc656b3c8e8f2966517c5ad40d1bfb6261dabf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Thu, 24 Aug 2023 21:21:10 +0200 Subject: [PATCH 23/71] refactor: replace all char pointers with std::string --- .../mods/autodownload/moddownloader.cpp | 18 +++++++++--------- NorthstarDLL/mods/autodownload/moddownloader.h | 12 ++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 54018cc07..adf3faeaa 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -88,7 +88,7 @@ size_t write_data(void* ptr, size_t size, size_t nmemb, FILE* stream) return written; } -fs::path ModDownloader::FetchModFromDistantStore(char* modName, char* modVersion) +fs::path ModDownloader::FetchModFromDistantStore(std::string modName, std::string modVersion) { // Build archive distant URI std::string archiveName = std::format("{}-{}.zip", verifiedMods[modName].dependencyPrefix, modVersion); @@ -134,13 +134,13 @@ fs::path ModDownloader::FetchModFromDistantStore(char* modName, char* modVersion return downloadPath; } -bool ModDownloader::IsModLegit(fs::path modPath, char* expectedChecksum) +bool ModDownloader::IsModLegit(fs::path modPath, std::string expectedChecksum) { // TODO implement return true; } -bool ModDownloader::IsModAuthorized(char* modName, char* modVersion) +bool ModDownloader::IsModAuthorized(std::string modName, std::string modVersion) { if (!verifiedMods.contains(modName)) { @@ -151,7 +151,7 @@ bool ModDownloader::IsModAuthorized(char* modName, char* modVersion) return versions.count(modVersion) != 0; } -void ModDownloader::DownloadMod(char* modName, char* modVersion) +void ModDownloader::DownloadMod(std::string modName, std::string modVersion) { // Check if mod can be auto-downloaded if (!IsModAuthorized(modName, modVersion)) @@ -166,7 +166,7 @@ void ModDownloader::DownloadMod(char* modName, char* modVersion) // Download mod archive std::string expectedHash = verifiedMods[modName].versions[modVersion].checksum; fs::path archiveLocation = FetchModFromDistantStore(modName, modVersion); - if (!IsModLegit(archiveLocation, (char*)expectedHash.c_str())) + if (!IsModLegit(archiveLocation, expectedHash)) { spdlog::warn("Archive hash does not match expected checksum, aborting."); return; @@ -200,8 +200,8 @@ void ConCommand_is_mod_verified(const CCommand& args) while (ss >> buf) tokens.push_back(buf); - char* modName = (char*)tokens[0].c_str(); - char* modVersion = (char*)tokens[1].c_str(); + std::string modName = tokens[0]; + std::string modVersion = tokens[1]; bool result = g_pModDownloader->IsModAuthorized(modName, modVersion); std::string msg = std::format("Mod \"{}\" (version {}) is verified: {}", modName, modVersion, result); spdlog::info(msg); @@ -221,8 +221,8 @@ void ConCommand_download_mod(const CCommand& args) while (ss >> buf) tokens.push_back(buf); - char* modName = (char*)tokens[0].c_str(); - char* modVersion = (char*)tokens[1].c_str(); + std::string modName = tokens[0]; + std::string modVersion = tokens[1]; g_pModDownloader->DownloadMod(modName, modVersion); } diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index fba053d25..9ec190198 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -26,7 +26,7 @@ class ModDownloader * @param modVersion version of the mod to be downloaded * @returns location of the downloaded archive */ - fs::path FetchModFromDistantStore(char* modName, char* modVersion); + fs::path FetchModFromDistantStore(std::string modName, std::string modVersion); /** * Tells if a mod archive has not been corrupted. @@ -40,7 +40,7 @@ class ModDownloader * @param expectedChecksum checksum the archive should have * @returns whether archive is legit */ - bool IsModLegit(fs::path modPath, char* expectedChecksum); + bool IsModLegit(fs::path modPath, std::string expectedChecksum); /** * Extracts a mod archive to the game folder. @@ -79,7 +79,7 @@ class ModDownloader * @param modVersion version of the mod to be downloaded * @returns nothing **/ - void DownloadMod(char* modName, char* modVersion); + void DownloadMod(std::string modName, std::string modVersion); /** * Checks whether a mod is verified. @@ -93,7 +93,7 @@ class ModDownloader * @param modVersion version of the mod to be checked, must follow semantic versioning * @returns whether the mod is authorized and can be auto-downloaded */ - bool IsModAuthorized(char* modName, char* modVersion); + bool IsModAuthorized(std::string modName, std::string modVersion); enum ModInstallState { @@ -130,7 +130,7 @@ class ModDownloader * @param modName name of the mod to get information of * @returns the class MOD_STATE instance */ - MOD_STATE GetModInstallProgress(char* modName); + MOD_STATE GetModInstallProgress(std::string modName); /** * Cancels installation of a mod. @@ -144,5 +144,5 @@ class ModDownloader * @param modName name of the mod to prevent installation of * @returns nothing */ - void CancelDownload(char* modName); + void CancelDownload(std::string modName); }; From 5b957f6570722bd2b331e4ade31629fea64cdc78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Fri, 25 Aug 2023 23:41:27 +0200 Subject: [PATCH 24/71] build: add minizip dependency --- .gitmodules | 3 +++ NorthstarDLL/CMakeLists.txt | 2 ++ cmake/Findminizip.cmake | 8 ++++++++ thirdparty/minizip | 1 + 4 files changed, 14 insertions(+) create mode 100644 cmake/Findminizip.cmake create mode 160000 thirdparty/minizip diff --git a/.gitmodules b/.gitmodules index 1806e83c7..c4cfc0a13 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,6 @@ path = thirdparty/minhook url = https://github.com/TsudaKageyu/minhook ignore = untracked +[submodule "thirdparty/minizip"] + path = thirdparty/minizip + url = https://github.com/zlib-ng/minizip-ng.git diff --git a/NorthstarDLL/CMakeLists.txt b/NorthstarDLL/CMakeLists.txt index 602a6da70..bdf758450 100644 --- a/NorthstarDLL/CMakeLists.txt +++ b/NorthstarDLL/CMakeLists.txt @@ -2,6 +2,7 @@ find_package(minhook REQUIRED) find_package(libcurl REQUIRED) +find_package(minizip REQUIRED) add_library(NorthstarDLL SHARED "client/audio.cpp" @@ -154,6 +155,7 @@ add_library(NorthstarDLL SHARED target_link_libraries(NorthstarDLL PRIVATE minhook libcurl + minizip WS2_32.lib Crypt32.lib Cryptui.lib diff --git a/cmake/Findminizip.cmake b/cmake/Findminizip.cmake new file mode 100644 index 000000000..151c97b43 --- /dev/null +++ b/cmake/Findminizip.cmake @@ -0,0 +1,8 @@ + +if(NOT minizip_FOUND) + check_init_submodule(${PROJECT_SOURCE_DIR}/thirdparty/minizip) + + add_subdirectory(${PROJECT_SOURCE_DIR}/thirdparty/minizip minizip) + set(minizip_FOUND 1 PARENT_SCOPE) +endif() + diff --git a/thirdparty/minizip b/thirdparty/minizip new file mode 160000 index 000000000..680d6f1dc --- /dev/null +++ b/thirdparty/minizip @@ -0,0 +1 @@ +Subproject commit 680d6f1dcf9de99fc033b54975a1dfff10be2b6b From 16bedc6b8658ce19f308202ebe653d044651498d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Fri, 25 Aug 2023 23:45:01 +0200 Subject: [PATCH 25/71] feat: list files in downloaded mod archive --- .../mods/autodownload/moddownloader.cpp | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index adf3faeaa..37eedb35d 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -1,5 +1,10 @@ #include "moddownloader.h" #include +#include +#include +#include +#include +#include ModDownloader* g_pModDownloader; @@ -151,6 +156,45 @@ bool ModDownloader::IsModAuthorized(std::string modName, std::string modVersion) return versions.count(modVersion) != 0; } +void ModDownloader::ExtractMod(fs::path modPath) +{ + unzFile file; + + file = unzOpen(modPath.generic_string().c_str()); + if (file == NULL) + { + spdlog::error("Cannot open archive located at {}.", modPath.generic_string()); + return; + } + + unz_global_info64 gi; + int status; + status = unzGetGlobalInfo64(file, &gi); + if (file != UNZ_OK) + { + spdlog::error("Failed getting information from archive (error code: {})", status); + } + + for (int i = 0; i < gi.number_entry; i++) + { + char filename_inzip[256]; + unz_file_info64 file_info; + status = unzGetCurrentFileInfo64(file, &file_info, filename_inzip, sizeof(filename_inzip), NULL, 0, NULL, 0); + spdlog::info("{}", filename_inzip); + + // Go to next file + if ((i + 1) < gi.number_entry) + { + status = unzGoToNextFile(file); + if (status != UNZ_OK) + { + spdlog::error("Error while browsing archive files (error code: {}).", status); + break; + } + } + } +} + void ModDownloader::DownloadMod(std::string modName, std::string modVersion) { // Check if mod can be auto-downloaded @@ -172,7 +216,8 @@ void ModDownloader::DownloadMod(std::string modName, std::string modVersion) return; } - // TODO extract mod archive + // Extract downloaded mod archive + ExtractMod(archiveLocation); REQUEST_END_CLEANUP: spdlog::info("ok"); From af92411ecae15349e730dfcb18c1e163bc698a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Sat, 26 Aug 2023 00:42:48 +0200 Subject: [PATCH 26/71] feat: compute mod files extraction destination --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 37eedb35d..f1597bd2a 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -175,12 +175,21 @@ void ModDownloader::ExtractMod(fs::path modPath) spdlog::error("Failed getting information from archive (error code: {})", status); } + // Mod directory name (removing the ".zip" fom the archive name) + std::string name = modPath.filename().string(); + name = name.substr(0, name.length() - 4); + fs::path modDirectory = GetRemoteModFolderPath() / name; + for (int i = 0; i < gi.number_entry; i++) { char filename_inzip[256]; unz_file_info64 file_info; status = unzGetCurrentFileInfo64(file, &file_info, filename_inzip, sizeof(filename_inzip), NULL, 0, NULL, 0); - spdlog::info("{}", filename_inzip); + + // Extract file + fs::path fileDestination = modDirectory / filename_inzip; + spdlog::info("{}", fileDestination.generic_string()); + // Go to next file if ((i + 1) < gi.number_entry) From 8a2e3ecf13700ff7d0a5dd60bef898274952610c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Sat, 26 Aug 2023 01:16:54 +0200 Subject: [PATCH 27/71] feat: create directories while extracting mod archive --- .../mods/autodownload/moddownloader.cpp | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index f1597bd2a..426efeacf 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -187,9 +187,39 @@ void ModDownloader::ExtractMod(fs::path modPath) status = unzGetCurrentFileInfo64(file, &file_info, filename_inzip, sizeof(filename_inzip), NULL, 0, NULL, 0); // Extract file - fs::path fileDestination = modDirectory / filename_inzip; - spdlog::info("{}", fileDestination.generic_string()); + { + std::error_code ec; + fs::path fileDestination = modDirectory / filename_inzip; + spdlog::info("{}", fileDestination.generic_string()); + + // Create parent directory if needed + if (!std::filesystem::exists(fileDestination.parent_path())) + { + spdlog::warn("Parent directory does not exist for file {}, creating it.", fileDestination.generic_string()); + if (!std::filesystem::create_directories(fileDestination.parent_path(), ec)) + { + spdlog::error("Parent directory ({}) creation failed.", fileDestination.parent_path().generic_string()); + return; + } + } + // If current file is a directory, create directory... + if (fileDestination.generic_string().back() == '/') + { + // Create directory + if (!std::filesystem::create_directory(fileDestination, ec)) + { + spdlog::error("Directory creation failed: {}", ec.message()); + return; + } + } + + // ...else create file + else + { + // TODO create file + } + } // Go to next file if ((i + 1) < gi.number_entry) From 85f416a79204becf8621b8740bca0b4cf896ab9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Sat, 26 Aug 2023 01:33:47 +0200 Subject: [PATCH 28/71] fix: do not stop mod archive extraction when trying to create an already-existing directory --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 426efeacf..436475700 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -196,7 +196,7 @@ void ModDownloader::ExtractMod(fs::path modPath) if (!std::filesystem::exists(fileDestination.parent_path())) { spdlog::warn("Parent directory does not exist for file {}, creating it.", fileDestination.generic_string()); - if (!std::filesystem::create_directories(fileDestination.parent_path(), ec)) + if (!std::filesystem::create_directories(fileDestination.parent_path(), ec) && ec.value() != 0) { spdlog::error("Parent directory ({}) creation failed.", fileDestination.parent_path().generic_string()); return; @@ -207,7 +207,7 @@ void ModDownloader::ExtractMod(fs::path modPath) if (fileDestination.generic_string().back() == '/') { // Create directory - if (!std::filesystem::create_directory(fileDestination, ec)) + if (!std::filesystem::create_directory(fileDestination, ec) && ec.value() != 0) { spdlog::error("Directory creation failed: {}", ec.message()); return; From 07e4c40846279cb48bdc3c0303d19a1f1ba40224 Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Sat, 26 Aug 2023 18:00:04 +0200 Subject: [PATCH 29/71] build: disable LZMA compression in minizip --- cmake/Findminizip.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/Findminizip.cmake b/cmake/Findminizip.cmake index 151c97b43..17489061b 100644 --- a/cmake/Findminizip.cmake +++ b/cmake/Findminizip.cmake @@ -2,6 +2,8 @@ if(NOT minizip_FOUND) check_init_submodule(${PROJECT_SOURCE_DIR}/thirdparty/minizip) + set(MZ_LZMA OFF CACHE BOOL "Disable LZMA & XZ compression") + add_subdirectory(${PROJECT_SOURCE_DIR}/thirdparty/minizip minizip) set(minizip_FOUND 1 PARENT_SCOPE) endif() From b22b06ff647a575d759661de1326ad0c9e0f2d5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Sat, 26 Aug 2023 21:48:44 +0200 Subject: [PATCH 30/71] feat: ensure files are present in zip archive --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 436475700..2f597bb4b 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -217,7 +217,12 @@ void ModDownloader::ExtractMod(fs::path modPath) // ...else create file else { - // TODO create file + // Ensure file is in zip archive + if (unzLocateFile(file, filename_inzip, 0) != UNZ_OK) + { + spdlog::error("File \"{}\" was not found in archive.", filename_inzip); + return; + } } } From b631df84b43ea70fd4701ea3b7c129e56d33445d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Tue, 29 Aug 2023 20:07:20 +0200 Subject: [PATCH 31/71] feat: add file creation skeleton --- .../mods/autodownload/moddownloader.cpp | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 2f597bb4b..4126ae800 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -223,6 +223,69 @@ void ModDownloader::ExtractMod(fs::path modPath) spdlog::error("File \"{}\" was not found in archive.", filename_inzip); return; } + + // Create file + int size_buf; + void* buf; + int err = UNZ_OK; + FILE* fout = NULL; + + // Create destination file + fout = fopen(fileDestination.generic_string().c_str(), "wb"); + if (fout == NULL) + { + spdlog::error("Failed creating destination file."); + return; + } + + // Allocate memory for buffer + size_buf = 8192; + buf = (void*)malloc(size_buf); + if (buf == NULL) + { + spdlog::error("Error while allocating memory."); + return; + } + + // Extract file to destination + do + { + err = unzReadCurrentFile(file, buf, size_buf); + if (err < 0) + { + spdlog::error("error {} with zipfile in unzReadCurrentFile", err); + break; + } + if (err > 0) + { + spdlog::info("oi oi oi {}", buf); + if (fwrite(buf, (unsigned)err, 1, fout) != 1) + { + spdlog::error("error in writing extracted file\n"); + err = UNZ_ERRNO; + break; + } + spdlog::info("fwrite result: {}", err); + } + } while (err > 0); + + if (err == UNZ_OK) + { + spdlog::info("CLOSING FILE"); + err = unzCloseCurrentFile(file); + if (err != UNZ_OK) + { + spdlog::error("error {} with zipfile in unzCloseCurrentFile", err); + } + } + else { + spdlog::info("yo? {}", err); + unzCloseCurrentFile(file); + } + + // Cleanup + if (fout) + fclose(fout); } } From 1e5878aa928f8910fdfdab70416952e5f77f23ed Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Tue, 29 Aug 2023 23:07:23 +0200 Subject: [PATCH 32/71] feat: open zip file to prepare its extraction --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 4126ae800..f5b9c11d6 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -230,6 +230,13 @@ void ModDownloader::ExtractMod(fs::path modPath) int err = UNZ_OK; FILE* fout = NULL; + // Open zip file to prepare its extraction + status = unzOpenCurrentFile(file); + if (status != UNZ_OK) + { + spdlog::error("Could not open file {} from archive.", filename_inzip); + } + // Create destination file fout = fopen(fileDestination.generic_string().c_str(), "wb"); if (fout == NULL) From b3b6ca4481d5aea684bc45996b1170f0c61d2f6e Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Tue, 29 Aug 2023 23:08:32 +0200 Subject: [PATCH 33/71] style: apply clang format --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index f5b9c11d6..176b2bd09 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -285,7 +285,8 @@ void ModDownloader::ExtractMod(fs::path modPath) spdlog::error("error {} with zipfile in unzCloseCurrentFile", err); } } - else { + else + { spdlog::info("yo? {}", err); unzCloseCurrentFile(file); } From 53e33efd6f61af2a83c8b6ce314051fa3d82e303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 30 Aug 2023 08:23:26 +0200 Subject: [PATCH 34/71] refactor: reorganize debug prints --- .../mods/autodownload/moddownloader.cpp | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 176b2bd09..5c5789dc3 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -190,12 +190,12 @@ void ModDownloader::ExtractMod(fs::path modPath) { std::error_code ec; fs::path fileDestination = modDirectory / filename_inzip; - spdlog::info("{}", fileDestination.generic_string()); + spdlog::info("=> {}", fileDestination.generic_string()); // Create parent directory if needed if (!std::filesystem::exists(fileDestination.parent_path())) { - spdlog::warn("Parent directory does not exist for file {}, creating it.", fileDestination.generic_string()); + spdlog::info("Parent directory does not exist, creating it.", fileDestination.generic_string()); if (!std::filesystem::create_directories(fileDestination.parent_path(), ec) && ec.value() != 0) { spdlog::error("Parent directory ({}) creation failed.", fileDestination.parent_path().generic_string()); @@ -265,30 +265,24 @@ void ModDownloader::ExtractMod(fs::path modPath) } if (err > 0) { - spdlog::info("oi oi oi {}", buf); if (fwrite(buf, (unsigned)err, 1, fout) != 1) { spdlog::error("error in writing extracted file\n"); err = UNZ_ERRNO; break; } - spdlog::info("fwrite result: {}", err); } } while (err > 0); - if (err == UNZ_OK) + if (err != UNZ_OK) { - spdlog::info("CLOSING FILE"); - err = unzCloseCurrentFile(file); - if (err != UNZ_OK) - { - spdlog::error("error {} with zipfile in unzCloseCurrentFile", err); - } + spdlog::error("An error occurred during file extraction (code: {})", err); + } - else + err = unzCloseCurrentFile(file); + if (err != UNZ_OK) { - spdlog::info("yo? {}", err); - unzCloseCurrentFile(file); + spdlog::error("error {} with zipfile in unzCloseCurrentFile", err); } // Cleanup @@ -335,7 +329,7 @@ void ModDownloader::DownloadMod(std::string modName, std::string modVersion) ExtractMod(archiveLocation); REQUEST_END_CLEANUP: - spdlog::info("ok"); + spdlog::info("Done downloading {}.", modName); }); requestThread.detach(); From 8e0a766d4ce8491c51ea8196fb0777447d7e24a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Wed, 30 Aug 2023 18:11:22 +0200 Subject: [PATCH 35/71] feat: enforce Thunderstore format for remote mods --- NorthstarDLL/mods/modmanager.cpp | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/NorthstarDLL/mods/modmanager.cpp b/NorthstarDLL/mods/modmanager.cpp index 044ec6d4d..8acda5f1a 100644 --- a/NorthstarDLL/mods/modmanager.cpp +++ b/NorthstarDLL/mods/modmanager.cpp @@ -612,32 +612,36 @@ void ModManager::LoadMods() // get mod directories std::filesystem::directory_iterator classicModsDir = fs::directory_iterator(GetModFolderPath()); std::filesystem::directory_iterator remoteModsDir = fs::directory_iterator(GetRemoteModFolderPath()); + std::filesystem::directory_iterator thunderstoreModsDir = fs::directory_iterator(GetThunderstoreModFolderPath()); for (std::filesystem::directory_iterator modIterator : {classicModsDir, remoteModsDir}) for (fs::directory_entry dir : modIterator) if (fs::exists(dir.path() / "mod.json")) modDirs.push_back(dir.path()); - // Special case for Thunderstore mods dir - std::filesystem::directory_iterator thunderstoreModsDir = fs::directory_iterator(GetThunderstoreModFolderPath()); + // Special case for Thunderstore and remote mods directories // Set up regex for `AUTHOR-MOD-VERSION` pattern std::regex pattern(R"(.*\\([a-zA-Z0-9_]+)-([a-zA-Z0-9_]+)-(\d+\.\d+\.\d+))"); - for (fs::directory_entry dir : thunderstoreModsDir) + + for (fs::directory_iterator dirIterator : {thunderstoreModsDir, remoteModsDir}) { - fs::path modsDir = dir.path() / "mods"; // Check for mods folder in the Thunderstore mod - // Use regex to match `AUTHOR-MOD-VERSION` pattern - if (!std::regex_match(dir.path().string(), pattern)) - { - spdlog::warn("The following directory did not match 'AUTHOR-MOD-VERSION': {}", dir.path().string()); - continue; // skip loading mod that doesn't match - } - if (fs::exists(modsDir) && fs::is_directory(modsDir)) + for (fs::directory_entry dir : dirIterator) { - for (fs::directory_entry subDir : fs::directory_iterator(modsDir)) + fs::path modsDir = dir.path() / "mods"; // Check for mods folder in the Thunderstore mod + // Use regex to match `AUTHOR-MOD-VERSION` pattern + if (!std::regex_match(dir.path().string(), pattern)) { - if (fs::exists(subDir.path() / "mod.json")) + spdlog::warn("The following directory did not match 'AUTHOR-MOD-VERSION': {}", dir.path().string()); + continue; // skip loading mod that doesn't match + } + if (fs::exists(modsDir) && fs::is_directory(modsDir)) + { + for (fs::directory_entry subDir : fs::directory_iterator(modsDir)) { - modDirs.push_back(subDir.path()); + if (fs::exists(subDir.path() / "mod.json")) + { + modDirs.push_back(subDir.path()); + } } } } From 9b68db0b42638e566b0e2991852096900764c94f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Thu, 31 Aug 2023 09:12:37 +0200 Subject: [PATCH 36/71] style: apply clang format --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 5c5789dc3..8a2488b10 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -277,7 +277,6 @@ void ModDownloader::ExtractMod(fs::path modPath) if (err != UNZ_OK) { spdlog::error("An error occurred during file extraction (code: {})", err); - } err = unzCloseCurrentFile(file); if (err != UNZ_OK) From 0c75bfcaf1c4eba8216a4c704e30e8695c7d0de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Thu, 31 Aug 2023 21:40:12 +0200 Subject: [PATCH 37/71] style: use camelCase everywhere --- .../mods/autodownload/moddownloader.cpp | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 8a2488b10..4944de0b7 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -14,7 +14,7 @@ ModDownloader::ModDownloader() modState = {}; } -size_t write_to_string(void* ptr, size_t size, size_t count, void* stream) +size_t writeToString(void* ptr, size_t size, size_t count, void* stream) { ((std::string*)stream)->append((char*)ptr, 0, size * count); return size * count; @@ -40,7 +40,7 @@ void ModDownloader::FetchModsListFromAPI() curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str()); curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, &readBuffer); - curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, write_to_string); + curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, writeToString); result = curl_easy_perform(easyhandle); if (result == CURLcode::CURLE_OK) @@ -86,7 +86,7 @@ void ModDownloader::FetchModsListFromAPI() requestThread.detach(); } -size_t write_data(void* ptr, size_t size, size_t nmemb, FILE* stream) +size_t writeData(void* ptr, size_t size, size_t nmemb, FILE* stream) { size_t written; written = fwrite(ptr, size, nmemb, stream); @@ -117,7 +117,7 @@ fs::path ModDownloader::FetchModFromDistantStore(std::string modName, std::strin curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str()); curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, fp); - curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, write_data); + curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, writeData); result = curl_easy_perform(easyhandle); if (result == CURLcode::CURLE_OK) @@ -182,14 +182,14 @@ void ModDownloader::ExtractMod(fs::path modPath) for (int i = 0; i < gi.number_entry; i++) { - char filename_inzip[256]; - unz_file_info64 file_info; - status = unzGetCurrentFileInfo64(file, &file_info, filename_inzip, sizeof(filename_inzip), NULL, 0, NULL, 0); + char zipFilename[256]; + unz_file_info64 fileInfo; + status = unzGetCurrentFileInfo64(file, &fileInfo, zipFilename, sizeof(zipFilename), NULL, 0, NULL, 0); // Extract file { std::error_code ec; - fs::path fileDestination = modDirectory / filename_inzip; + fs::path fileDestination = modDirectory / zipFilename; spdlog::info("=> {}", fileDestination.generic_string()); // Create parent directory if needed @@ -218,15 +218,15 @@ void ModDownloader::ExtractMod(fs::path modPath) else { // Ensure file is in zip archive - if (unzLocateFile(file, filename_inzip, 0) != UNZ_OK) + if (unzLocateFile(file, zipFilename, 0) != UNZ_OK) { - spdlog::error("File \"{}\" was not found in archive.", filename_inzip); + spdlog::error("File \"{}\" was not found in archive.", zipFilename); return; } // Create file - int size_buf; - void* buf; + int bufferSize; + void* buffer; int err = UNZ_OK; FILE* fout = NULL; @@ -234,7 +234,7 @@ void ModDownloader::ExtractMod(fs::path modPath) status = unzOpenCurrentFile(file); if (status != UNZ_OK) { - spdlog::error("Could not open file {} from archive.", filename_inzip); + spdlog::error("Could not open file {} from archive.", zipFilename); } // Create destination file @@ -246,9 +246,9 @@ void ModDownloader::ExtractMod(fs::path modPath) } // Allocate memory for buffer - size_buf = 8192; - buf = (void*)malloc(size_buf); - if (buf == NULL) + bufferSize = 8192; + buffer = (void*)malloc(bufferSize); + if (buffer == NULL) { spdlog::error("Error while allocating memory."); return; @@ -257,7 +257,7 @@ void ModDownloader::ExtractMod(fs::path modPath) // Extract file to destination do { - err = unzReadCurrentFile(file, buf, size_buf); + err = unzReadCurrentFile(file, buffer, bufferSize); if (err < 0) { spdlog::error("error {} with zipfile in unzReadCurrentFile", err); @@ -265,7 +265,7 @@ void ModDownloader::ExtractMod(fs::path modPath) } if (err > 0) { - if (fwrite(buf, (unsigned)err, 1, fout) != 1) + if (fwrite(buffer, (unsigned)err, 1, fout) != 1) { spdlog::error("error in writing extracted file\n"); err = UNZ_ERRNO; @@ -334,12 +334,12 @@ void ModDownloader::DownloadMod(std::string modName, std::string modVersion) requestThread.detach(); } -void ConCommand_fetch_verified_mods(const CCommand& args) +void ConCommandFetchVerifiedMods(const CCommand& args) { g_pModDownloader->FetchModsListFromAPI(); } -void ConCommand_is_mod_verified(const CCommand& args) +void ConCommandIsModVerified(const CCommand& args) { if (args.ArgC() < 3) { @@ -347,11 +347,11 @@ void ConCommand_is_mod_verified(const CCommand& args) } // Split arguments string by whitespaces (https://stackoverflow.com/a/5208977) - std::string buf; + std::string buffer; std::stringstream ss(args.ArgS()); std::vector tokens; - while (ss >> buf) - tokens.push_back(buf); + while (ss >> buffer) + tokens.push_back(buffer); std::string modName = tokens[0]; std::string modVersion = tokens[1]; @@ -360,7 +360,7 @@ void ConCommand_is_mod_verified(const CCommand& args) spdlog::info(msg); } -void ConCommand_download_mod(const CCommand& args) +void ConCommandDownloadMod(const CCommand& args) { if (args.ArgC() < 3) { @@ -368,11 +368,11 @@ void ConCommand_download_mod(const CCommand& args) } // Split arguments string by whitespaces (https://stackoverflow.com/a/5208977) - std::string buf; + std::string buffer; std::stringstream ss(args.ArgS()); std::vector tokens; - while (ss >> buf) - tokens.push_back(buf); + while (ss >> buffer) + tokens.push_back(buffer); std::string modName = tokens[0]; std::string modVersion = tokens[1]; @@ -383,7 +383,7 @@ ON_DLL_LOAD_RELIESON("engine.dll", ModDownloader, (ConCommand), (CModule module) { g_pModDownloader = new ModDownloader(); RegisterConCommand( - "fetch_verified_mods", ConCommand_fetch_verified_mods, "fetches verified mods list from GitHub repository", FCVAR_NONE); - RegisterConCommand("is_mod_verified", ConCommand_is_mod_verified, "checks if a mod is included in verified mods list", FCVAR_NONE); - RegisterConCommand("download_mod", ConCommand_download_mod, "downloads a mod from remote store", FCVAR_NONE); + "fetch_verified_mods", ConCommandFetchVerifiedMods, "fetches verified mods list from GitHub repository", FCVAR_NONE); + RegisterConCommand("is_mod_verified", ConCommandIsModVerified, "checks if a mod is included in verified mods list", FCVAR_NONE); + RegisterConCommand("download_mod", ConCommandDownloadMod, "downloads a mod from remote store", FCVAR_NONE); } From 741c9f58ca1971627f65c89f5039a9f8322bb370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Thu, 31 Aug 2023 21:44:44 +0200 Subject: [PATCH 38/71] style: apply clang format --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 4944de0b7..e6c6b4209 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -382,8 +382,7 @@ void ConCommandDownloadMod(const CCommand& args) ON_DLL_LOAD_RELIESON("engine.dll", ModDownloader, (ConCommand), (CModule module)) { g_pModDownloader = new ModDownloader(); - RegisterConCommand( - "fetch_verified_mods", ConCommandFetchVerifiedMods, "fetches verified mods list from GitHub repository", FCVAR_NONE); + RegisterConCommand("fetch_verified_mods", ConCommandFetchVerifiedMods, "fetches verified mods list from GitHub repository", FCVAR_NONE); RegisterConCommand("is_mod_verified", ConCommandIsModVerified, "checks if a mod is included in verified mods list", FCVAR_NONE); RegisterConCommand("download_mod", ConCommandDownloadMod, "downloads a mod from remote store", FCVAR_NONE); } From 79d08129776f7b00b40ebb835866355fbe9cb4a7 Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Thu, 31 Aug 2023 23:27:55 +0200 Subject: [PATCH 39/71] refactor: set IsModAuthorized function as private and remove associated ConCommand --- .../mods/autodownload/moddownloader.cpp | 22 --------------- .../mods/autodownload/moddownloader.h | 28 +++++++++---------- 2 files changed, 14 insertions(+), 36 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index e6c6b4209..fe3c13db9 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -339,27 +339,6 @@ void ConCommandFetchVerifiedMods(const CCommand& args) g_pModDownloader->FetchModsListFromAPI(); } -void ConCommandIsModVerified(const CCommand& args) -{ - if (args.ArgC() < 3) - { - return; - } - - // Split arguments string by whitespaces (https://stackoverflow.com/a/5208977) - std::string buffer; - std::stringstream ss(args.ArgS()); - std::vector tokens; - while (ss >> buffer) - tokens.push_back(buffer); - - std::string modName = tokens[0]; - std::string modVersion = tokens[1]; - bool result = g_pModDownloader->IsModAuthorized(modName, modVersion); - std::string msg = std::format("Mod \"{}\" (version {}) is verified: {}", modName, modVersion, result); - spdlog::info(msg); -} - void ConCommandDownloadMod(const CCommand& args) { if (args.ArgC() < 3) @@ -383,6 +362,5 @@ ON_DLL_LOAD_RELIESON("engine.dll", ModDownloader, (ConCommand), (CModule module) { g_pModDownloader = new ModDownloader(); RegisterConCommand("fetch_verified_mods", ConCommandFetchVerifiedMods, "fetches verified mods list from GitHub repository", FCVAR_NONE); - RegisterConCommand("is_mod_verified", ConCommandIsModVerified, "checks if a mod is included in verified mods list", FCVAR_NONE); RegisterConCommand("download_mod", ConCommandDownloadMod, "downloads a mod from remote store", FCVAR_NONE); } diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index 9ec190198..51e8ecf31 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -28,6 +28,20 @@ class ModDownloader */ fs::path FetchModFromDistantStore(std::string modName, std::string modVersion); + /** + * Checks whether a mod is verified. + * + * A mod is deemed verified/authorized through a manual validation process that is + * described here: https://github.com/R2Northstar/VerifiedMods; in practice, a mod + * is considered authorized if their name AND exact version appear in the + * `verifiedMods` variable. + * + * @param modName name of the mod to be checked + * @param modVersion version of the mod to be checked, must follow semantic versioning + * @returns whether the mod is authorized and can be auto-downloaded + */ + bool IsModAuthorized(std::string modName, std::string modVersion); + /** * Tells if a mod archive has not been corrupted. * @@ -81,20 +95,6 @@ class ModDownloader **/ void DownloadMod(std::string modName, std::string modVersion); - /** - * Checks whether a mod is verified. - * - * A mod is deemed verified/authorized through a manual validation process that is - * described here: https://github.com/R2Northstar/VerifiedMods; in practice, a mod - * is considered authorized if their name AND exact version appear in the - * `verifiedMods` variable. - * - * @param modName name of the mod to be checked - * @param modVersion version of the mod to be checked, must follow semantic versioning - * @returns whether the mod is authorized and can be auto-downloaded - */ - bool IsModAuthorized(std::string modName, std::string modVersion); - enum ModInstallState { // Normal installation process From ff40df556c5dcc5431a10d1ddb4ae8f27ca3f0ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 4 Sep 2023 20:14:01 +0200 Subject: [PATCH 40/71] feat: add a flag to bypass mod verification process --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 14 +++++++++++++- NorthstarDLL/mods/autodownload/moddownloader.h | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index e6c6b4209..9c2e09dcc 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -141,12 +141,24 @@ fs::path ModDownloader::FetchModFromDistantStore(std::string modName, std::strin bool ModDownloader::IsModLegit(fs::path modPath, std::string expectedChecksum) { + if (strstr(GetCommandLineA(), VERIFICATION_FLAG)) + { + spdlog::info("Bypassing mod verification due to flag set up."); + return true; + } + // TODO implement - return true; + return false; } bool ModDownloader::IsModAuthorized(std::string modName, std::string modVersion) { + if (strstr(GetCommandLineA(), VERIFICATION_FLAG)) + { + spdlog::info("Bypassing mod verification due to flag set up."); + return true; + } + if (!verifiedMods.contains(modName)) { return false; diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index 9ec190198..de4954239 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -3,6 +3,7 @@ class ModDownloader private: const char* STORE_URL = "https://gcdn.thunderstore.io/live/repository/packages/"; const char* MODS_LIST_URL = "https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json"; + const char* VERIFICATION_COMMAND = "-disablemodverification"; struct VerifiedModVersion { From efb3d89f61d049cb066a8b34f66079448d0a08fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 4 Sep 2023 20:22:09 +0200 Subject: [PATCH 41/71] fix: rename flag in header --- NorthstarDLL/mods/autodownload/moddownloader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index de4954239..5691afca5 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -3,7 +3,7 @@ class ModDownloader private: const char* STORE_URL = "https://gcdn.thunderstore.io/live/repository/packages/"; const char* MODS_LIST_URL = "https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json"; - const char* VERIFICATION_COMMAND = "-disablemodverification"; + const char* VERIFICATION_FLAG = "-disablemodverification"; struct VerifiedModVersion { From 3b0c502b7ad0965b931767a43209640b4475d49f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 4 Sep 2023 21:33:29 +0200 Subject: [PATCH 42/71] feat: remove mod archive after download --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 9c2e09dcc..8478977dd 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -313,6 +313,8 @@ void ModDownloader::ExtractMod(fs::path modPath) } } } + + unzClose(file); } void ModDownloader::DownloadMod(std::string modName, std::string modVersion) @@ -340,6 +342,15 @@ void ModDownloader::DownloadMod(std::string modName, std::string modVersion) ExtractMod(archiveLocation); REQUEST_END_CLEANUP: + try + { + remove( archiveLocation ); + } + catch (const std::exception& a) + { + spdlog::error("Error while removing downloaded archive: {}", a.what()); + } + spdlog::info("Done downloading {}.", modName); }); From a3d87f28a4e16d6a0755d9b41718e3bda5f105c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 4 Sep 2023 21:40:32 +0200 Subject: [PATCH 43/71] fix: remove mod archive when archive checksum verification fails --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 8478977dd..e4f04acac 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -335,7 +335,7 @@ void ModDownloader::DownloadMod(std::string modName, std::string modVersion) if (!IsModLegit(archiveLocation, expectedHash)) { spdlog::warn("Archive hash does not match expected checksum, aborting."); - return; + goto REQUEST_END_CLEANUP; } // Extract downloaded mod archive From e737d382f220a758e1fd56ea47fef272cbd7bfc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 4 Sep 2023 21:55:12 +0200 Subject: [PATCH 44/71] feat: add a clean subroutine to the ExtractMod function --- .../mods/autodownload/moddownloader.cpp | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index e4f04acac..cf52fbc91 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -171,12 +171,14 @@ bool ModDownloader::IsModAuthorized(std::string modName, std::string modVersion) void ModDownloader::ExtractMod(fs::path modPath) { unzFile file; + std::string name; + fs::path modDirectory; file = unzOpen(modPath.generic_string().c_str()); if (file == NULL) { spdlog::error("Cannot open archive located at {}.", modPath.generic_string()); - return; + goto EXTRACTION_CLEANUP; } unz_global_info64 gi; @@ -188,9 +190,9 @@ void ModDownloader::ExtractMod(fs::path modPath) } // Mod directory name (removing the ".zip" fom the archive name) - std::string name = modPath.filename().string(); + name = modPath.filename().string(); name = name.substr(0, name.length() - 4); - fs::path modDirectory = GetRemoteModFolderPath() / name; + modDirectory = GetRemoteModFolderPath() / name; for (int i = 0; i < gi.number_entry; i++) { @@ -211,7 +213,7 @@ void ModDownloader::ExtractMod(fs::path modPath) if (!std::filesystem::create_directories(fileDestination.parent_path(), ec) && ec.value() != 0) { spdlog::error("Parent directory ({}) creation failed.", fileDestination.parent_path().generic_string()); - return; + goto EXTRACTION_CLEANUP; } } @@ -222,7 +224,7 @@ void ModDownloader::ExtractMod(fs::path modPath) if (!std::filesystem::create_directory(fileDestination, ec) && ec.value() != 0) { spdlog::error("Directory creation failed: {}", ec.message()); - return; + goto EXTRACTION_CLEANUP; } } @@ -233,7 +235,7 @@ void ModDownloader::ExtractMod(fs::path modPath) if (unzLocateFile(file, zipFilename, 0) != UNZ_OK) { spdlog::error("File \"{}\" was not found in archive.", zipFilename); - return; + goto EXTRACTION_CLEANUP; } // Create file @@ -247,6 +249,7 @@ void ModDownloader::ExtractMod(fs::path modPath) if (status != UNZ_OK) { spdlog::error("Could not open file {} from archive.", zipFilename); + goto EXTRACTION_CLEANUP; } // Create destination file @@ -254,7 +257,7 @@ void ModDownloader::ExtractMod(fs::path modPath) if (fout == NULL) { spdlog::error("Failed creating destination file."); - return; + goto EXTRACTION_CLEANUP; } // Allocate memory for buffer @@ -263,7 +266,7 @@ void ModDownloader::ExtractMod(fs::path modPath) if (buffer == NULL) { spdlog::error("Error while allocating memory."); - return; + goto EXTRACTION_CLEANUP; } // Extract file to destination @@ -289,6 +292,7 @@ void ModDownloader::ExtractMod(fs::path modPath) if (err != UNZ_OK) { spdlog::error("An error occurred during file extraction (code: {})", err); + goto EXTRACTION_CLEANUP; } err = unzCloseCurrentFile(file); if (err != UNZ_OK) @@ -309,12 +313,16 @@ void ModDownloader::ExtractMod(fs::path modPath) if (status != UNZ_OK) { spdlog::error("Error while browsing archive files (error code: {}).", status); - break; + goto EXTRACTION_CLEANUP; } } } - unzClose(file); +EXTRACTION_CLEANUP: + if (unzClose(file) != MZ_OK) + { + spdlog::error("Failed closing mod archive after extraction."); + } } void ModDownloader::DownloadMod(std::string modName, std::string modVersion) From c64aaab6393d6cfc55e623bbac7378d4463dae6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 4 Sep 2023 22:27:21 +0200 Subject: [PATCH 45/71] fix: disable curl verbose mode --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index cf52fbc91..29530772f 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -38,7 +38,6 @@ void ModDownloader::FetchModsListFromAPI() curl_easy_setopt(easyhandle, CURLOPT_CUSTOMREQUEST, "GET"); curl_easy_setopt(easyhandle, CURLOPT_TIMEOUT, 30L); curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str()); - curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, &readBuffer); curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, writeToString); result = curl_easy_perform(easyhandle); @@ -115,7 +114,6 @@ fs::path ModDownloader::FetchModFromDistantStore(std::string modName, std::strin curl_easy_setopt(easyhandle, CURLOPT_TIMEOUT, 30L); curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str()); - curl_easy_setopt(easyhandle, CURLOPT_VERBOSE, 1L); curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, fp); curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, writeData); result = curl_easy_perform(easyhandle); From 6f5c2f2e3e85e2c0eec9f5eca0254e7e1c39d5fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 4 Sep 2023 22:27:46 +0200 Subject: [PATCH 46/71] style: apply clang format --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 29530772f..20e337f21 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -350,7 +350,7 @@ void ModDownloader::DownloadMod(std::string modName, std::string modVersion) REQUEST_END_CLEANUP: try { - remove( archiveLocation ); + remove(archiveLocation); } catch (const std::exception& a) { From 24bb06e4a16d0e692072df14833390e8fd397866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Tue, 5 Sep 2023 07:46:42 +0200 Subject: [PATCH 47/71] fix: fail mod fetching HTTP request if status >= 400 --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 20e337f21..f25020a0c 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -38,6 +38,7 @@ void ModDownloader::FetchModsListFromAPI() curl_easy_setopt(easyhandle, CURLOPT_CUSTOMREQUEST, "GET"); curl_easy_setopt(easyhandle, CURLOPT_TIMEOUT, 30L); curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str()); + curl_easy_setopt(easyhandle, CURLOPT_FAILONERROR, 1L); curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, &readBuffer); curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, writeToString); result = curl_easy_perform(easyhandle); From ea3a10a87c98d8232b94c8114386061d334a944f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Thu, 7 Sep 2023 18:17:20 +0200 Subject: [PATCH 48/71] fix: mod archive fetching failure is now tolerated --- .../mods/autodownload/moddownloader.cpp | 86 ++++++++++++------- .../mods/autodownload/moddownloader.h | 7 +- 2 files changed, 58 insertions(+), 35 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index f25020a0c..2020c2803 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -5,6 +5,11 @@ #include #include #include +#include +#include +#include +#include +#include ModDownloader* g_pModDownloader; @@ -93,7 +98,40 @@ size_t writeData(void* ptr, size_t size, size_t nmemb, FILE* stream) return written; } -fs::path ModDownloader::FetchModFromDistantStore(std::string modName, std::string modVersion) +void func(std::promise>&& p, std::string url, fs::path downloadPath) +{ + bool failed = false; + FILE* fp = fopen(downloadPath.generic_string().c_str(), "wb"); + CURLcode result; + CURL* easyhandle; + easyhandle = curl_easy_init(); + + curl_easy_setopt(easyhandle, CURLOPT_TIMEOUT, 30L); + curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str()); + curl_easy_setopt(easyhandle, CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, writeData); + result = curl_easy_perform(easyhandle); + + if (result == CURLcode::CURLE_OK) + { + spdlog::info("Mod archive successfully fetched."); + goto REQUEST_END_CLEANUP; + } + else + { + spdlog::error("Fetching mod archive failed: {}", curl_easy_strerror(result)); + failed = true; + goto REQUEST_END_CLEANUP; + } + +REQUEST_END_CLEANUP: + curl_easy_cleanup(easyhandle); + fclose(fp); + p.set_value(failed ? std::optional() : std::optional(downloadPath)); +} + +std::optional ModDownloader::FetchModFromDistantStore(std::string modName, std::string modVersion) { // Build archive distant URI std::string archiveName = std::format("{}-{}.zip", verifiedMods[modName].dependencyPrefix, modVersion); @@ -105,37 +143,11 @@ fs::path ModDownloader::FetchModFromDistantStore(std::string modName, std::strin spdlog::info(std::format("Downloading archive to {}", downloadPath.generic_string())); // Download the actual archive - std::thread requestThread( - [this, url, downloadPath]() - { - FILE* fp = fopen(downloadPath.generic_string().c_str(), "wb"); - CURLcode result; - CURL* easyhandle; - easyhandle = curl_easy_init(); - - curl_easy_setopt(easyhandle, CURLOPT_TIMEOUT, 30L); - curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str()); - curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, fp); - curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, writeData); - result = curl_easy_perform(easyhandle); - - if (result == CURLcode::CURLE_OK) - { - spdlog::info("Mod archive successfully fetched."); - } - else - { - spdlog::error("Fetching mod archive failed: {}", curl_easy_strerror(result)); - goto REQUEST_END_CLEANUP; - } - - REQUEST_END_CLEANUP: - curl_easy_cleanup(easyhandle); - fclose(fp); - }); - - requestThread.join(); - return downloadPath; + std::promise> promise; + auto f = promise.get_future(); + std::thread t(&func, std::move(promise), url, downloadPath); + t.join(); + return f.get(); } bool ModDownloader::IsModLegit(fs::path modPath, std::string expectedChecksum) @@ -336,9 +348,17 @@ void ModDownloader::DownloadMod(std::string modName, std::string modVersion) std::thread requestThread( [this, modName, modVersion]() { + fs::path archiveLocation; + // Download mod archive std::string expectedHash = verifiedMods[modName].versions[modVersion].checksum; - fs::path archiveLocation = FetchModFromDistantStore(modName, modVersion); + std::optional fetchingResult = FetchModFromDistantStore(modName, modVersion); + if (!fetchingResult.has_value()) + { + spdlog::error("Something went wrong while fetching archive, aborting."); + goto REQUEST_END_CLEANUP; + } + archiveLocation = fetchingResult.value(); if (!IsModLegit(archiveLocation, expectedHash)) { spdlog::warn("Archive hash does not match expected checksum, aborting."); diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index 5691afca5..1b6eb604e 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -20,14 +20,17 @@ class ModDownloader * Downloads a mod archive from distant store. * * This rebuilds the URI of the mod archive using both a predefined store URI - * and the mod dependency string from the `verifiedMods` variabl; fetched + * and the mod dependency string from the `verifiedMods` variable; fetched * archive is then stored in a temporary location. * + * If something went wrong during archive download, this will return an empty + * optional object. + * * @param modName name of the mod to be downloaded * @param modVersion version of the mod to be downloaded * @returns location of the downloaded archive */ - fs::path FetchModFromDistantStore(std::string modName, std::string modVersion); + std::optional FetchModFromDistantStore(std::string modName, std::string modVersion); /** * Tells if a mod archive has not been corrupted. From 82427b8d85e3475afe88ec910265376cb955fba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Thu, 7 Sep 2023 18:18:03 +0200 Subject: [PATCH 49/71] feat: add IsModLegit function skeleton --- .../mods/autodownload/moddownloader.cpp | 99 ++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 2020c2803..ca3fd17ab 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -158,7 +158,104 @@ bool ModDownloader::IsModLegit(fs::path modPath, std::string expectedChecksum) return true; } - // TODO implement + NTSTATUS status; + BCRYPT_ALG_HANDLE algorithmHandle = NULL; + BCRYPT_HASH_HANDLE hashHandle = NULL; + std::vector hash; + DWORD hashLength = 0; + DWORD resultLength = 0; + + constexpr size_t bufferSize {1 << 12}; + std::vector buffer(bufferSize, '\0'); + std::ifstream fp(modPath.generic_string(), std::ios::binary); + + // Open an algorithm handle + // This sample passes BCRYPT_HASH_REUSABLE_FLAG with BCryptAlgorithmProvider(...) to load a provider which supports reusable hash + status = BCryptOpenAlgorithmProvider( + &algorithmHandle, // Alg Handle pointer + BCRYPT_SHA256_ALGORITHM, // Cryptographic Algorithm name (null terminated unicode string) + NULL, // Provider name; if null, the default provider is loaded + BCRYPT_HASH_REUSABLE_FLAG); // Flags; Loads a provider which supports reusable hash + if (!NT_SUCCESS(status)) + { + goto cleanup; + } + + // Obtain the length of the hash + status = BCryptGetProperty( + algorithmHandle, // Handle to a CNG object + BCRYPT_HASH_LENGTH, // Property name (null terminated unicode string) + (PBYTE)&hashLength, // Address of the output buffer which recieves the property value + sizeof(hashLength), // Size of the buffer in bytes + &resultLength, // Number of bytes that were copied into the buffer + 0); // Flags + if (!NT_SUCCESS(status)) + { + // goto cleanup; + return false; + } + + // Create a hash handle + status = BCryptCreateHash( + algorithmHandle, // Handle to an algorithm provider + &hashHandle, // A pointer to a hash handle - can be a hash or hmac object + NULL, // Pointer to the buffer that recieves the hash/hmac object + 0, // Size of the buffer in bytes + NULL, // A pointer to a key to use for the hash or MAC + 0, // Size of the key in bytes + 0); // Flags + if (!NT_SUCCESS(status)) + { + goto cleanup; + } + + // Hash archive content + if (!fp.is_open()) + { + spdlog::error("Unable to open archive."); + return false; + } + while (fp.good()) + { + fp.read(buffer.data(), bufferSize); + status = BCryptHashData(hashHandle, (PBYTE)buffer.data(), bufferSize, 0); + if (!NT_SUCCESS(status)) + { + goto cleanup; + } + } + + hash = std::vector(hashLength); + + // Obtain the hash of the message(s) into the hash buffer + status = BCryptFinishHash( + hashHandle, // Handle to the hash or MAC object + hash.data(), // A pointer to a buffer that receives the hash or MAC value + hashLength, // Size of the buffer in bytes + 0); // Flags + if (!NT_SUCCESS(status)) + { + goto cleanup; + } + + spdlog::info("Expected checksum: {}", expectedChecksum); + spdlog::info("Computed checksum: {}", (char*)hash.data()); + + // TODO compare hash + +cleanup: + if (NULL != hashHandle) + { + BCryptDestroyHash(hashHandle); // Handle to hash/MAC object which needs to be destroyed + } + + if (NULL != algorithmHandle) + { + BCryptCloseAlgorithmProvider( + algorithmHandle, // Handle to the algorithm provider which needs to be closed + 0); // Flags + } + return false; } From c4c9c4193a28ee6c3af5782b7138f571f2f3e24e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Thu, 7 Sep 2023 18:18:50 +0200 Subject: [PATCH 50/71] style: apply clang format --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index ca3fd17ab..7109987aa 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -195,8 +195,8 @@ bool ModDownloader::IsModLegit(fs::path modPath, std::string expectedChecksum) return false; } - // Create a hash handle - status = BCryptCreateHash( + // Create a hash handle + status = BCryptCreateHash( algorithmHandle, // Handle to an algorithm provider &hashHandle, // A pointer to a hash handle - can be a hash or hmac object NULL, // Pointer to the buffer that recieves the hash/hmac object From 61efbc211fb0c8ce33901f33b96c9bdb0dd3031f Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Sun, 10 Sep 2023 15:12:47 +0200 Subject: [PATCH 51/71] fix: do not try to always read bufferSize bytes --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 61f0532a9..9db18d3aa 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -215,13 +215,18 @@ bool ModDownloader::IsModLegit(fs::path modPath, std::string expectedChecksum) spdlog::error("Unable to open archive."); return false; } + fp.seekg(0, fp.beg); while (fp.good()) { fp.read(buffer.data(), bufferSize); - status = BCryptHashData(hashHandle, (PBYTE)buffer.data(), bufferSize, 0); - if (!NT_SUCCESS(status)) + std::streamsize bytesRead = fp.gcount(); + if (bytesRead > 0) { - goto cleanup; + status = BCryptHashData(hashHandle, (PBYTE)buffer.data(), bytesRead, 0); + if (!NT_SUCCESS(status)) + { + goto cleanup; + } } } From 1e6c04e8600531375be1b71630605f0baaef2f3a Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Sun, 10 Sep 2023 15:26:22 +0200 Subject: [PATCH 52/71] fix: convert hash to string using bytes raw values --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 9db18d3aa..5b6b042ed 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -164,6 +164,7 @@ bool ModDownloader::IsModLegit(fs::path modPath, std::string expectedChecksum) std::vector hash; DWORD hashLength = 0; DWORD resultLength = 0; + std::stringstream ss; constexpr size_t bufferSize {1 << 12}; std::vector buffer(bufferSize, '\0'); @@ -243,10 +244,16 @@ bool ModDownloader::IsModLegit(fs::path modPath, std::string expectedChecksum) goto cleanup; } - spdlog::info("Expected checksum: {}", expectedChecksum); - spdlog::info("Computed checksum: {}", (char*)hash.data()); + // Convert hash to string using bytes raw values + ss << std::hex << std::setfill('0'); + for (int i = 0; i < hashLength; i++) + { + ss << std::hex << std::setw(2) << static_cast(hash.data()[i]); + } - // TODO compare hash + spdlog::info("Expected checksum: {}", expectedChecksum); + spdlog::info("Computed checksum: {}", ss.str()); + return expectedChecksum.compare(ss.str()) == 0; cleanup: if (NULL != hashHandle) From 15043b6d01d80c889d412cc46a4eef57ce4175ea Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Sun, 10 Sep 2023 16:34:13 +0200 Subject: [PATCH 53/71] style: move files in CMakeLists.txt --- NorthstarDLL/CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/NorthstarDLL/CMakeLists.txt b/NorthstarDLL/CMakeLists.txt index bdf758450..d7f144f6e 100644 --- a/NorthstarDLL/CMakeLists.txt +++ b/NorthstarDLL/CMakeLists.txt @@ -73,6 +73,8 @@ add_library(NorthstarDLL SHARED "logging/sourceconsole.h" "masterserver/masterserver.cpp" "masterserver/masterserver.h" + "mods/autodownload/moddownloader.h" + "mods/autodownload/moddownloader.cpp" "mods/compiled/kb_act.cpp" "mods/compiled/modkeyvalues.cpp" "mods/compiled/modpdef.cpp" @@ -149,8 +151,7 @@ add_library(NorthstarDLL SHARED "audio_asm.asm" "dllmain.cpp" "dllmain.h" - "ns_version.h" - "mods/autodownload/moddownloader.h" "mods/autodownload/moddownloader.cpp") + "ns_version.h") target_link_libraries(NorthstarDLL PRIVATE minhook From 5576aa9332858f0c77c410219593b8c77cf4b5bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 11 Sep 2023 08:14:46 +0200 Subject: [PATCH 54/71] refactor: pass string views to FetchModFromDistantStore --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 12 ++++++------ NorthstarDLL/mods/autodownload/moddownloader.h | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 5b6b042ed..b858e62cd 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -98,7 +98,7 @@ size_t writeData(void* ptr, size_t size, size_t nmemb, FILE* stream) return written; } -void func(std::promise>&& p, std::string url, fs::path downloadPath) +void func(std::promise>&& p, std::string_view url, fs::path downloadPath) { bool failed = false; FILE* fp = fopen(downloadPath.generic_string().c_str(), "wb"); @@ -107,7 +107,7 @@ void func(std::promise>&& p, std::string url, fs::path d easyhandle = curl_easy_init(); curl_easy_setopt(easyhandle, CURLOPT_TIMEOUT, 30L); - curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str()); + curl_easy_setopt(easyhandle, CURLOPT_URL, url.data()); curl_easy_setopt(easyhandle, CURLOPT_FAILONERROR, 1L); curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, fp); curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, writeData); @@ -131,10 +131,10 @@ void func(std::promise>&& p, std::string url, fs::path d p.set_value(failed ? std::optional() : std::optional(downloadPath)); } -std::optional ModDownloader::FetchModFromDistantStore(std::string modName, std::string modVersion) +std::optional ModDownloader::FetchModFromDistantStore(std::string_view modName, std::string_view modVersion) { // Build archive distant URI - std::string archiveName = std::format("{}-{}.zip", verifiedMods[modName].dependencyPrefix, modVersion); + std::string archiveName = std::format("{}-{}.zip", verifiedMods[modName.data()].dependencyPrefix, modVersion.data()); std::string url = STORE_URL + archiveName; spdlog::info(std::format("Fetching mod archive from {}", url)); @@ -145,7 +145,7 @@ std::optional ModDownloader::FetchModFromDistantStore(std::string modN // Download the actual archive std::promise> promise; auto f = promise.get_future(); - std::thread t(&func, std::move(promise), url, downloadPath); + std::thread t(&func, std::move(promise), std::string_view(url), downloadPath); t.join(); return f.get(); } @@ -461,7 +461,7 @@ void ModDownloader::DownloadMod(std::string modName, std::string modVersion) // Download mod archive std::string expectedHash = verifiedMods[modName].versions[modVersion].checksum; - std::optional fetchingResult = FetchModFromDistantStore(modName, modVersion); + std::optional fetchingResult = FetchModFromDistantStore(std::string_view(modName), std::string_view(modVersion)); if (!fetchingResult.has_value()) { spdlog::error("Something went wrong while fetching archive, aborting."); diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index 206571f20..a6aa95124 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -30,7 +30,7 @@ class ModDownloader * @param modVersion version of the mod to be downloaded * @returns location of the downloaded archive */ - std::optional FetchModFromDistantStore(std::string modName, std::string modVersion); + std::optional FetchModFromDistantStore(std::string_view modName, std::string_view modVersion); /** * Checks whether a mod is verified. From 471baf72547a66666f0502f1ecd706209a89cf55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 11 Sep 2023 08:20:39 +0200 Subject: [PATCH 55/71] refactor: pass string views to IsModAuthorized --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 10 +++++----- NorthstarDLL/mods/autodownload/moddownloader.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index b858e62cd..ac05462e8 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -271,7 +271,7 @@ bool ModDownloader::IsModLegit(fs::path modPath, std::string expectedChecksum) return false; } -bool ModDownloader::IsModAuthorized(std::string modName, std::string modVersion) +bool ModDownloader::IsModAuthorized(std::string_view modName, std::string_view modVersion) { if (strstr(GetCommandLineA(), VERIFICATION_FLAG)) { @@ -279,13 +279,13 @@ bool ModDownloader::IsModAuthorized(std::string modName, std::string modVersion) return true; } - if (!verifiedMods.contains(modName)) + if (!verifiedMods.contains(modName.data())) { return false; } - std::unordered_map versions = verifiedMods[modName].versions; - return versions.count(modVersion) != 0; + std::unordered_map versions = verifiedMods[modName.data()].versions; + return versions.count(modVersion.data()) != 0; } void ModDownloader::ExtractMod(fs::path modPath) @@ -448,7 +448,7 @@ void ModDownloader::ExtractMod(fs::path modPath) void ModDownloader::DownloadMod(std::string modName, std::string modVersion) { // Check if mod can be auto-downloaded - if (!IsModAuthorized(modName, modVersion)) + if (!IsModAuthorized(std::string_view(modName), std::string_view(modVersion))) { spdlog::warn("Tried to download a mod that is not verified, aborting."); return; diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index a6aa95124..6682e79b9 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -44,7 +44,7 @@ class ModDownloader * @param modVersion version of the mod to be checked, must follow semantic versioning * @returns whether the mod is authorized and can be auto-downloaded */ - bool IsModAuthorized(std::string modName, std::string modVersion); + bool IsModAuthorized(std::string_view modName, std::string_view modVersion); /** * Tells if a mod archive has not been corrupted. From d6d2630b6c9afffdb4b11cb737f7f8c2d62e8b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 11 Sep 2023 08:29:57 +0200 Subject: [PATCH 56/71] refactor: pass string view to IsModLegit --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 6 +++--- NorthstarDLL/mods/autodownload/moddownloader.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index ac05462e8..c0a7b64a7 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -150,7 +150,7 @@ std::optional ModDownloader::FetchModFromDistantStore(std::string_view return f.get(); } -bool ModDownloader::IsModLegit(fs::path modPath, std::string expectedChecksum) +bool ModDownloader::IsModLegit(fs::path modPath, std::string_view expectedChecksum) { if (strstr(GetCommandLineA(), VERIFICATION_FLAG)) { @@ -251,7 +251,7 @@ bool ModDownloader::IsModLegit(fs::path modPath, std::string expectedChecksum) ss << std::hex << std::setw(2) << static_cast(hash.data()[i]); } - spdlog::info("Expected checksum: {}", expectedChecksum); + spdlog::info("Expected checksum: {}", expectedChecksum.data()); spdlog::info("Computed checksum: {}", ss.str()); return expectedChecksum.compare(ss.str()) == 0; @@ -468,7 +468,7 @@ void ModDownloader::DownloadMod(std::string modName, std::string modVersion) goto REQUEST_END_CLEANUP; } archiveLocation = fetchingResult.value(); - if (!IsModLegit(archiveLocation, expectedHash)) + if (!IsModLegit(archiveLocation, std::string_view(expectedHash))) { spdlog::warn("Archive hash does not match expected checksum, aborting."); goto REQUEST_END_CLEANUP; diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index 6682e79b9..728b15ee0 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -58,7 +58,7 @@ class ModDownloader * @param expectedChecksum checksum the archive should have * @returns whether archive is legit */ - bool IsModLegit(fs::path modPath, std::string expectedChecksum); + bool IsModLegit(fs::path modPath, std::string_view expectedChecksum); /** * Extracts a mod archive to the game folder. From e8bbfe87698e602629417364a07ea0211830487d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Thu, 14 Sep 2023 23:36:50 +0200 Subject: [PATCH 57/71] style: update NorthstarDLL/CMakeLists.txt Co-authored-by: GeckoEidechse <40122905+GeckoEidechse@users.noreply.github.com> --- NorthstarDLL/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NorthstarDLL/CMakeLists.txt b/NorthstarDLL/CMakeLists.txt index d7f144f6e..e418f371c 100644 --- a/NorthstarDLL/CMakeLists.txt +++ b/NorthstarDLL/CMakeLists.txt @@ -151,7 +151,8 @@ add_library(NorthstarDLL SHARED "audio_asm.asm" "dllmain.cpp" "dllmain.h" - "ns_version.h") + "ns_version.h" +) target_link_libraries(NorthstarDLL PRIVATE minhook From 6d094c1a84331189a9b7eaf26d3ac18a3eddd184 Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Thu, 14 Sep 2023 23:37:52 +0200 Subject: [PATCH 58/71] style: replace tabs with spaces in NorthstarDLL/CMakeLists.txt --- NorthstarDLL/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NorthstarDLL/CMakeLists.txt b/NorthstarDLL/CMakeLists.txt index e418f371c..d22a7e3f7 100644 --- a/NorthstarDLL/CMakeLists.txt +++ b/NorthstarDLL/CMakeLists.txt @@ -157,7 +157,7 @@ add_library(NorthstarDLL SHARED target_link_libraries(NorthstarDLL PRIVATE minhook libcurl - minizip + minizip WS2_32.lib Crypt32.lib Cryptui.lib From 1f3d9544baf817946fc1b3ece8b66d640572d0c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Sun, 24 Sep 2023 12:57:49 +0200 Subject: [PATCH 59/71] style: remove typo Co-authored-by: Jack <66967891+ASpoonPlaysGames@users.noreply.github.com> --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index c0a7b64a7..96d1c8a13 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -15,7 +15,7 @@ ModDownloader* g_pModDownloader; ModDownloader::ModDownloader() { - spdlog::info("Mod downloaded initialized"); + spdlog::info("Mod downloader initialized"); modState = {}; } From 377ca2c716aba2ed20ed1be067089fe5171c819e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Sun, 24 Sep 2023 13:03:53 +0200 Subject: [PATCH 60/71] style: rewrite else case Co-authored-by: Jack <66967891+ASpoonPlaysGames@users.noreply.github.com> --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 96d1c8a13..5cc4bf168 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -347,7 +347,6 @@ void ModDownloader::ExtractMod(fs::path modPath) goto EXTRACTION_CLEANUP; } } - // ...else create file else { From 7b38a4b60eb550df630506cdf689b94fab99165c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Sun, 24 Sep 2023 13:56:15 +0200 Subject: [PATCH 61/71] style: use PascalCase for function names Co-authored-by: Jack <66967891+ASpoonPlaysGames@users.noreply.github.com> --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 5cc4bf168..25a4d866f 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -19,7 +19,7 @@ ModDownloader::ModDownloader() modState = {}; } -size_t writeToString(void* ptr, size_t size, size_t count, void* stream) +size_t WriteToString(void* ptr, size_t size, size_t count, void* stream) { ((std::string*)stream)->append((char*)ptr, 0, size * count); return size * count; @@ -45,7 +45,7 @@ void ModDownloader::FetchModsListFromAPI() curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str()); curl_easy_setopt(easyhandle, CURLOPT_FAILONERROR, 1L); curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, &readBuffer); - curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, writeToString); + curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, WriteToString); result = curl_easy_perform(easyhandle); if (result == CURLcode::CURLE_OK) @@ -91,14 +91,14 @@ void ModDownloader::FetchModsListFromAPI() requestThread.detach(); } -size_t writeData(void* ptr, size_t size, size_t nmemb, FILE* stream) +size_t WriteData(void* ptr, size_t size, size_t nmemb, FILE* stream) { size_t written; written = fwrite(ptr, size, nmemb, stream); return written; } -void func(std::promise>&& p, std::string_view url, fs::path downloadPath) +void FetchModSync(std::promise>&& p, std::string_view url, fs::path downloadPath) { bool failed = false; FILE* fp = fopen(downloadPath.generic_string().c_str(), "wb"); @@ -110,7 +110,7 @@ void func(std::promise>&& p, std::string_view url, fs::p curl_easy_setopt(easyhandle, CURLOPT_URL, url.data()); curl_easy_setopt(easyhandle, CURLOPT_FAILONERROR, 1L); curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, fp); - curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, writeData); + curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, WriteData); result = curl_easy_perform(easyhandle); if (result == CURLcode::CURLE_OK) @@ -145,7 +145,7 @@ std::optional ModDownloader::FetchModFromDistantStore(std::string_view // Download the actual archive std::promise> promise; auto f = promise.get_future(); - std::thread t(&func, std::move(promise), std::string_view(url), downloadPath); + std::thread t(&FetchModSync, std::move(promise), std::string_view(url), downloadPath); t.join(); return f.get(); } From 99127c1833cd36d30187f77eabdc2bb0e8dc891a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Sun, 24 Sep 2023 14:03:10 +0200 Subject: [PATCH 62/71] refactor: initialize bufferSize on declaration Co-authored-by: Jack <66967891+ASpoonPlaysGames@users.noreply.github.com> --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 25a4d866f..0cec8f32d 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -358,7 +358,7 @@ void ModDownloader::ExtractMod(fs::path modPath) } // Create file - int bufferSize; + int bufferSize = 8192; void* buffer; int err = UNZ_OK; FILE* fout = NULL; @@ -380,7 +380,6 @@ void ModDownloader::ExtractMod(fs::path modPath) } // Allocate memory for buffer - bufferSize = 8192; buffer = (void*)malloc(bufferSize); if (buffer == NULL) { From 915583d88e2f367aa46c5591f7e31f19e4845ba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Sun, 24 Sep 2023 14:06:17 +0200 Subject: [PATCH 63/71] refactor: bufferSize is const --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 0cec8f32d..e7b9a77d2 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -358,7 +358,7 @@ void ModDownloader::ExtractMod(fs::path modPath) } // Create file - int bufferSize = 8192; + const int bufferSize = 8192; void* buffer; int err = UNZ_OK; FILE* fout = NULL; From 4f2394e4bef0650c56c9b790dffc497669341ee0 Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Mon, 25 Sep 2023 16:51:55 +0200 Subject: [PATCH 64/71] refactor: move modState initialization to header --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 1 - NorthstarDLL/mods/autodownload/moddownloader.h | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index e7b9a77d2..1f8d28305 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -16,7 +16,6 @@ ModDownloader* g_pModDownloader; ModDownloader::ModDownloader() { spdlog::info("Mod downloader initialized"); - modState = {}; } size_t WriteToString(void* ptr, size_t size, size_t count, void* stream) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index 728b15ee0..c189230a5 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -123,7 +123,7 @@ class ModDownloader int progress; int total; float ratio; - } modState; + } modState = {}; /** * Retrieves installation information of a mod. From 84805764566da587e2846cb9297b34438a44dcb2 Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Fri, 13 Oct 2023 17:58:14 +0200 Subject: [PATCH 65/71] fix: check archive information status Co-authored-by: Jan --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index 1f8d28305..db35c8cb3 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -303,7 +303,7 @@ void ModDownloader::ExtractMod(fs::path modPath) unz_global_info64 gi; int status; status = unzGetGlobalInfo64(file, &gi); - if (file != UNZ_OK) + if (status != UNZ_OK) { spdlog::error("Failed getting information from archive (error code: {})", status); } From 059bb6e6599cfae48f894c98e22f955fe8ea067f Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Fri, 13 Oct 2023 18:20:22 +0200 Subject: [PATCH 66/71] fix: abort mod extraction if information cannot be retrieved from archive Co-authored-by: Jan --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index db35c8cb3..ef832595d 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -306,6 +306,7 @@ void ModDownloader::ExtractMod(fs::path modPath) if (status != UNZ_OK) { spdlog::error("Failed getting information from archive (error code: {})", status); + goto EXTRACTION_CLEANUP; } // Mod directory name (removing the ".zip" fom the archive name) From 4dbaa470e41f0151cebecfa12ab6a1dee0005ab6 Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Sun, 15 Oct 2023 22:40:21 +0200 Subject: [PATCH 67/71] refactor: use GetArray method to read mod versions Co-authored-by: Jack <66967891+ASpoonPlaysGames@users.noreply.github.com> --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index ef832595d..e859de6f5 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -68,9 +68,8 @@ void ModDownloader::FetchModsListFromAPI() std::unordered_map modVersions; rapidjson::Value& versions = i->value["Versions"]; assert(versions.IsArray()); - for (rapidjson::Value::ConstValueIterator itr = versions.Begin(); itr != versions.End(); ++itr) + for (auto& attribute : versions.GetArray()) { - const rapidjson::Value& attribute = *itr; assert(attribute.IsObject()); std::string version = attribute["Version"].GetString(); std::string checksum = attribute["Checksum"].GetString(); From 7548c47415fc76fbb9e9b7bce9e774ac0425cbab Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Mon, 16 Oct 2023 00:25:15 +0200 Subject: [PATCH 68/71] fix: use mod name as dependency prefix with bypass flag --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 4 +++- NorthstarDLL/mods/autodownload/moddownloader.h | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index e859de6f5..e8af44951 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -131,8 +131,10 @@ void FetchModSync(std::promise>&& p, std::string_view ur std::optional ModDownloader::FetchModFromDistantStore(std::string_view modName, std::string_view modVersion) { + // Retrieve mod prefix from local mods list, or use mod name as mod prefix if bypass flag is set + std::string modPrefix = strstr(GetCommandLineA(), VERIFICATION_FLAG) ? modName.data() : verifiedMods[modName.data()].dependencyPrefix; // Build archive distant URI - std::string archiveName = std::format("{}-{}.zip", verifiedMods[modName.data()].dependencyPrefix, modVersion.data()); + std::string archiveName = std::format("{}-{}.zip", modPrefix, modVersion.data()); std::string url = STORE_URL + archiveName; spdlog::info(std::format("Fetching mod archive from {}", url)); diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index c189230a5..090b6f0d9 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -20,7 +20,8 @@ class ModDownloader * Downloads a mod archive from distant store. * * This rebuilds the URI of the mod archive using both a predefined store URI - * and the mod dependency string from the `verifiedMods` variable; fetched + * and the mod dependency string from the `verifiedMods` variable, or using + * input mod name as mod dependency string if bypass flag is set up; fetched * archive is then stored in a temporary location. * * If something went wrong during archive download, this will return an empty From 8a1792c5c19e8b206f5e77b40a8b5df49afa2753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Mon, 16 Oct 2023 20:54:03 +0200 Subject: [PATCH 69/71] feat: add a command line arg to specify verified mods list URL --- .../mods/autodownload/moddownloader.cpp | 30 ++++++++++++++++++- .../mods/autodownload/moddownloader.h | 6 ++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp index e8af44951..9c1489c65 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.cpp +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -16,6 +16,34 @@ ModDownloader* g_pModDownloader; ModDownloader::ModDownloader() { spdlog::info("Mod downloader initialized"); + + // Initialise mods list URI + char* clachar = strstr(GetCommandLineA(), CUSTOM_MODS_URL_FLAG); + if (clachar) + { + std::string url; + int iFlagStringLength = strlen(CUSTOM_MODS_URL_FLAG); + std::string cla = std::string(clachar); + if (strncmp(cla.substr(iFlagStringLength, 1).c_str(), "\"", 1)) + { + int space = cla.find(" "); + url = cla.substr(iFlagStringLength, space - iFlagStringLength); + } + else + { + std::string quote = "\""; + int quote1 = cla.find(quote); + int quote2 = (cla.substr(quote1 + 1)).find(quote); + url = cla.substr(quote1 + 1, quote2); + } + spdlog::info("Found custom verified mods URL in command line argument: {}", url); + modsListUrl = strdup(url.c_str()); + } + else + { + spdlog::info("Custom verified mods URL not found in command line arguments, using default URL."); + modsListUrl = strdup(DEFAULT_MODS_LIST_URL); + } } size_t WriteToString(void* ptr, size_t size, size_t count, void* stream) @@ -32,7 +60,7 @@ void ModDownloader::FetchModsListFromAPI() CURLcode result; CURL* easyhandle; rapidjson::Document verifiedModsJson; - std::string url = MODS_LIST_URL; + std::string url = modsListUrl; curl_global_init(CURL_GLOBAL_ALL); easyhandle = curl_easy_init(); diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index 090b6f0d9..974a5fbf6 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -1,9 +1,11 @@ class ModDownloader { private: - const char* STORE_URL = "https://gcdn.thunderstore.io/live/repository/packages/"; - const char* MODS_LIST_URL = "https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json"; const char* VERIFICATION_FLAG = "-disablemodverification"; + const char* CUSTOM_MODS_URL_FLAG = "-customverifiedurl="; + const char* STORE_URL = "https://gcdn.thunderstore.io/live/repository/packages/"; + const char* DEFAULT_MODS_LIST_URL = "https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json"; + char* modsListUrl; struct VerifiedModVersion { From bd350b591c153aa6b05a40ee9aadda408bf3cc9a Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Tue, 17 Oct 2023 00:17:33 +0200 Subject: [PATCH 70/71] refactor: remove useless GetModInstallProgress method signature --- NorthstarDLL/mods/autodownload/moddownloader.h | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index 974a5fbf6..c67602c37 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -128,17 +128,6 @@ class ModDownloader float ratio; } modState = {}; - /** - * Retrieves installation information of a mod. - * - * Since mods are installed one at a time, mod installation data is stored in a global - * MOD_STATE instance. - * - * @param modName name of the mod to get information of - * @returns the class MOD_STATE instance - */ - MOD_STATE GetModInstallProgress(std::string modName); - /** * Cancels installation of a mod. * From ee3d3ff392babc2f33525056e04fc751c3abb3ad Mon Sep 17 00:00:00 2001 From: Remy Raes Date: Tue, 17 Oct 2023 00:23:06 +0200 Subject: [PATCH 71/71] refactor: CancelDownload method takes no input argument This method does not need a mod name as argument, as mods are downloaded one by one. --- NorthstarDLL/mods/autodownload/moddownloader.h | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h index c67602c37..edaf090bf 100644 --- a/NorthstarDLL/mods/autodownload/moddownloader.h +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -129,16 +129,13 @@ class ModDownloader } modState = {}; /** - * Cancels installation of a mod. + * Cancels installation of the mod. * - * Prevents installation of a mod currently being installed, no matter the install + * Prevents installation of the mod currently being installed, no matter the install * progress (downloading, checksuming, extracting), and frees all resources currently * being used in this purpose. * - * Does nothing if mod passed as argument is not being installed. - * - * @param modName name of the mod to prevent installation of * @returns nothing */ - void CancelDownload(std::string modName); + void CancelDownload(); };