Compress demos to nearly 10% of original.

docs/pages/todo/brainstorm_now.md

@@ -10,7 +10,6 @@ summary: That which we are brainstorming at the moment.
 # BEFORE STEAM
 
 - fix small capsule
-- Demos should be lz4 compressed and keep max 100 mb worth maybe
 
 # REST
 

docs/pages/todo/todo_done.md

@@ -6582,3 +6582,7 @@ This will discard your redo history."
     - i'd unassign them
 
 - internal addresses are fucked up again, no port shown and doesnt detect it
+- Demos should be lz4 compressed and keep max 100 mb worth maybe
+    - This can be done asynchronously in main menu, exactly like with neon map regeneration
+    - It's good because it won't delay getting back to the main menu when client is exited, plus it will work even if the client crashes and doesn't get the chance to compress
+

src/application/gui/demo_chooser.h

@@ -52,7 +52,7 @@ class demo_chooser : keyboard_acquiring_popup {
 				all_paths.clear();
 
 				auto path_adder = [this](const auto& full_path) {
-					if (full_path.extension() != ".dem") {
+					if (full_path.extension() != ".demc") {
 						return callback_result::CONTINUE;
 					}
 
@@ -63,11 +63,11 @@ class demo_chooser : keyboard_acquiring_popup {
 
 					try {
 						auto t = augs::open_binary_input_stream(full_path);
-						decltype(demo_file_meta::server_name) name;
-						augs::read_bytes(t, name);
+						demo_file_meta file_meta;
+						augs::read_bytes(t, file_meta);
 
-						meta.server_name = name;
-						meta.write_time = augs::date_time(augs::last_write_time(full_path)).how_long_ago();
+						meta.server_name = file_meta.server_name;
+						meta.write_time = augs::date_time::from_utc_timestamp(file_meta.when_recorded).how_long_ago();
 					}
 					catch (const std::ifstream::failure&) {
 

src/application/setups/client/client_setup.cpp

@@ -48,13 +48,41 @@
 #include "augs/readwrite/json_readwrite_errors.h"
 #include "application/setups/server/file_chunk_packet.h"
 #include "application/setups/client/direct_file_download.hpp"
+#include "augs/misc/compress.h"
 
 void client_demo_player::play_demo_from(const augs::path_type& p) {
 	source_path = p;
-	auto source = augs::open_binary_input_stream(source_path);
 
+	auto source_bytes = augs::file_to_bytes(source_path);
+	auto source = augs::make_ptr_read_stream(source_bytes);
 	augs::read_bytes(source, meta);
-	augs::read_vector_until_eof(source, demo_steps);
+
+	const auto pos = source.get_read_pos();
+
+	if (pos < source_bytes.size()) {
+		std::vector<std::byte> decompressed;
+		decompressed.resize(meta.uncompressed_size);
+
+		augs::decompress(
+			source_bytes.data() + pos,
+			source_bytes.size() - pos,
+			decompressed
+		);
+
+		auto s = augs::make_ptr_read_stream(decompressed);
+
+		try {
+			while (s.get_read_pos() <= decompressed.size()) {
+				demo_step step;
+				augs::read_bytes(s, step);
+				demo_steps.emplace_back(std::move(step));
+			}
+		}
+		catch (...) {
+
+		}
+	}
+
 
 	gui.open();
 }
@@ -163,6 +191,7 @@ void client_setup::flush_demo_steps() {
 				meta.server_name = displayed_connecting_server_name;
 				meta.server_address = last_addr.address;
 				meta.version = hypersomnia_version();
+				meta.when_recorded = augs::date_time().get_utc_timestamp();
 				augs::write_bytes(out, meta);
 
 				const auto version_info_path = augs::path_type(recorded_demo_path).replace_extension(".version.txt");

src/application/setups/client/demo_file_meta.h

@@ -8,5 +8,7 @@ struct demo_file_meta {
 	server_name_type server_name;
 	address_string_type server_address;
 	hypersomnia_version version;
+	version_timestamp_string when_recorded;
+	std::size_t uncompressed_size = 0;
 	// END GEN INTROSPECTOR
 };

src/augs/misc/compress.cpp

@@ -31,19 +31,28 @@ namespace augs {
 		const std::vector<std::byte>& input,
 		std::vector<std::byte>& output
 	) {
+		compress(state, input.data(), input.size(), output);
+	}
+
+	void compress(
+		std::vector<std::byte>& state,
+		const std::byte* input,
+		const std::size_t input_size,
+		std::vector<std::byte>& output
+	) {
 #if DISABLE_COMPRESSION
 		(void)state;
-		concatenate(output, input);
+		output.insert(output.end(), input, input + input_size);
 #else
-		const auto size_bound = LZ4_compressBound(input.size());
+		const auto size_bound = LZ4_compressBound(input_size);
 		const auto prev_size = output.size();
 		output.resize(prev_size + size_bound);
 
 		const auto bytes_written = LZ4_compress_fast_extState(
 			reinterpret_cast<void*>(state.data()), 
-			reinterpret_cast<const char*>(input.data()), 
+			reinterpret_cast<const char*>(input), 
 			reinterpret_cast<char*>(output.data() + prev_size), 
-			input.size(),
+			input_size,
 			size_bound,
 			1
 		);
@@ -89,12 +98,12 @@ namespace augs {
 
 		if (bytes_read < 0) {
 			output.clear();
-			throw decompression_error("Decompression failure. Failed to read any bytes.");
+			throw decompression_error("CHECK IF YOU PASSED CORRECT uncompressed_size! Decompression failure. Failed to read any bytes.");
 		}
 
 		if (uncompressed_size != static_cast<std::size_t>(bytes_read)) {
 			output.clear();
-			throw decompression_error("Decompression failure. Read %x bytes, but expected %x.", bytes_read, uncompressed_size);
+			throw decompression_error("CHECK IF YOU PASSED CORRECT uncompressed_size! Decompression failure. Read %x bytes, but expected %x.", bytes_read, uncompressed_size);
 		}
 #endif
 	}

src/augs/misc/compress.h

@@ -20,6 +20,13 @@ namespace augs {
 		std::vector<std::byte>& output
 	);
 
+	void compress(
+		std::vector<std::byte>& state,
+		const std::byte* input,
+		const std::size_t input_size,
+		std::vector<std::byte>& output
+	);
+
 	std::vector<std::byte> decompress(
 		const std::vector<std::byte>& input,
 		std::size_t uncompressed_size

src/view/viewables/streaming/viewables_streaming.cpp

@@ -12,6 +12,11 @@
 #include "augs/misc/imgui/imgui_control_wrappers.h"
 #include "augs/misc/imgui/imgui_scope_wrappers.h"
 #include "augs/filesystem/file.h"
+#include "augs/filesystem/directory.h"
+#include "augs/misc/compress.h"
+#include "augs/readwrite/byte_file.h"
+#include "augs/readwrite/to_bytes.h"
+#include "application/setups/client/demo_file_meta.h"
 
 void viewables_streaming::request_rescan() {
 	if (!general_atlas.empty()) {
@@ -338,6 +343,10 @@ void texture_in_progress<output_type>::finalize_load(augs::renderer& renderer) {
 }
 
 void viewables_streaming::finalize_load(viewables_finalize_input in) {
+	if (valid_and_is_ready(future_compressed_demos)) {
+		future_compressed_demos.get();
+	}
+
 	const auto current_frame = in.current_frame;
 	auto& now_all_defs = now_loaded_viewables_defs;
 
@@ -428,6 +437,82 @@ bool viewables_streaming::general_atlas_in_progress() const {
 	return general_atlas_progress.has_value();
 }
 
+void viewables_streaming::wait_demos_compressed() {
+	if (future_compressed_demos.valid()) {
+		future_compressed_demos.get();
+	}
+}
+
+void viewables_streaming::recompress_demos() {
+	future_compressed_demos = launch_async([]() {
+		std::vector<augs::path_type> file_list;
+
+		augs::for_each_in_directory(
+			augs::path_type(DEMOS_DIR),
+			[](auto&&...) { return callback_result::CONTINUE; },
+			[&file_list](const augs::path_type& demo) {
+				if (demo.extension() == ".dem") {
+					file_list.push_back(demo);
+				}
+
+				return callback_result::CONTINUE;
+			}
+		);
+
+		if (file_list.size() > 0) {
+			LOG("%x demos to be compressed.", file_list.size());
+		}
+		else {
+			LOG("All demos are already compressed.");
+		}
+
+		auto state = augs::make_compression_state();
+
+		for (const auto& demo_path : file_list) {
+			LOG("Compressing: %x", demo_path);
+			auto contents = augs::file_to_bytes(demo_path);
+
+			demo_file_meta meta;
+
+			auto s = augs::make_ptr_read_stream(contents);
+			try {
+				augs::read_bytes(s, meta);
+			}
+			catch (...) {
+				/* Broken/deprecated demo. Just delete it and move on. */
+
+				augs::remove_file(demo_path);
+				continue;
+			}
+
+			const auto pos = s.get_read_pos();
+
+			if (pos == contents.size()) {
+				LOG("Demo was empty.");
+				continue;
+			}
+
+			meta.uncompressed_size = contents.size() - pos;
+			auto compressed = augs::to_bytes(meta);
+
+			augs::compress(
+				state,
+				contents.data() + pos,
+				contents.size() - pos,
+				compressed
+			);
+
+			auto new_path = demo_path;
+			new_path.replace_extension(".demc");
+
+			augs::bytes_to_file(compressed, new_path);
+			augs::remove_file(demo_path);
+		}
+
+		return true;
+	});
+}
+
 void viewables_streaming::display_loading_progress() const {
 	using namespace augs::imgui;
 
@@ -470,4 +555,12 @@ void viewables_streaming::display_loading_progress() const {
 			ImGui::ProgressBar(progress_percent, ImVec2(-1.0f,0.0f));
 		}
 	}
+	else {
+		if (future_compressed_demos.valid()) {
+			center_next_window(ImGuiCond_Always);
+			auto loading_window = scoped_window("Demo file compression in progress", nullptr, ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoFocusOnAppearing);
+
+			text_color("The game is compressing demo files. Please be patient.\n", yellow);
+		}
+	}
 }

src/view/viewables/streaming/viewables_streaming.h

@@ -78,6 +78,8 @@ struct texture_in_progress {
 };
 
 class viewables_streaming {
+	std::future<bool> future_compressed_demos;
+
 	std::vector<rgba> general_atlas_pbo_fallback;
 
 	all_loaded_gui_fonts loaded_gui_fonts;
@@ -141,6 +143,8 @@ class viewables_streaming {
 	void display_loading_progress() const;
 
 	void request_rescan();
+	void recompress_demos();
+	void wait_demos_compressed();
 
 	auto& get_general_atlas() {
 		return general_atlas;

src/work.cpp

@@ -1260,12 +1260,15 @@ work_result work(const int argc, const char* const * const argv) try {
 			});
 
 			map_catalogue_gui.request_refresh();
+			streaming.recompress_demos();
 		}
 	};
 
 	auto launch_client = [&](const bool ignore_nat_check) {
 		bool public_internet = false;
 
+		streaming.wait_demos_compressed();
+
 		if (ignore_nat_check) {
 			LOG("Finished NAT traversal. Connecting immediately.");
 		}