add zstd compression

cmake/dependencies.cmake

@@ -51,6 +51,31 @@ else()
     import_pkgconfig_target(TARGET_NAME xz PKGCONFIG_TARGET liblzma)
 endif()
 
+set(USE_SYSTEM_ZSTD OFF CACHE BOOL "Use system zstd instead of building our own")
+
+if(NOT USE_SYSTEM_ZSTD)
+    message(STATUS "Downloading and building zstd")
+
+    ExternalProject_Add(zstd-EXTERNAL
+        URL https://github.com/facebook/zstd/releases/download/v1.5.0/zstd-1.5.0.tar.gz
+        URL_HASH SHA512=b322fc1b89a556827b7fece2fb0f34c83bf65bb85b2468c791d6d9178a65c81e21f4171b7533cbf12bc1dfb2fd323d3e8c34d86167b157645c27f65186eec659
+        CONFIGURE_COMMAND echo configure zstd
+        BUILD_COMMAND CC=${CC} CXX=${CXX} CFLAGS=-fPIC CPPFLAGS=${CPPFLAGS} LDFLAGS=${LDFLAGS} ${MAKE} -C<SOURCE_DIR>/lib ZSTD_LIB_MINIFY=1 libzstd.a
+        INSTALL_COMMAND ${MAKE} -C<SOURCE_DIR>/lib PREFIX=<INSTALL_DIR> install-static install-includes
+    )
+
+    import_external_project(
+        TARGET_NAME zstd
+        EXT_PROJECT_NAME zstd-EXTERNAL
+        LIBRARY_DIRS <SOURCE_DIR>/lib/
+        LIBRARIES "<SOURCE_DIR>/lib/libzstd.a"
+        INCLUDE_DIRS "<SOURCE_DIR>/lib/"
+    )
+else()
+    message(STATUS "Using system zstd")
+
+    import_pkgconfig_target(TARGET_NAME zstd PKGCONFIG_TARGET libzstd)
+endif()
 
 # as distros don't provide suitable squashfuse and squashfs-tools, those dependencies are bundled in, can, and should
 # be used from this repository for AppImageKit
@@ -83,8 +108,9 @@ if(NOT USE_SYSTEM_SQUASHFUSE)
         COMMAND ${AUTORECONF} -fi || true
         COMMAND ${SED} -i "/PKG_CHECK_MODULES.*/,/,:./d" configure  # https://github.com/vasi/squashfuse/issues/12
         COMMAND ${SED} -i "s/typedef off_t sqfs_off_t/typedef int64_t sqfs_off_t/g" common.h  # off_t's size might differ, see https://stackoverflow.com/a/9073762
-        COMMAND CC=${CC} CXX=${CXX} CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} <SOURCE_DIR>/configure --disable-demo --disable-high-level --without-lzo --without-lz4 --prefix=<INSTALL_DIR> --libdir=<INSTALL_DIR>/lib --with-xz=${xz_PREFIX} ${EXTRA_CONFIGURE_FLAGS}
+        COMMAND CC=${CC} CXX=${CXX} CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} <SOURCE_DIR>/configure --disable-demo --disable-high-level --without-lzo --without-lz4 --prefix=<INSTALL_DIR> --libdir=<INSTALL_DIR>/lib --with-xz=${xz_PREFIX} --with-zstd=${zstd_PREFIX} ${EXTRA_CONFIGURE_FLAGS}
         COMMAND ${SED} -i "s|XZ_LIBS = -llzma |XZ_LIBS = -Bstatic ${xz_LIBRARIES}/|g" Makefile
+        COMMAND ${SED} -i "s|ZSTD_LIBS = |ZSTD_LIBS = -Bstatic ${zstd_LIBRARIES}|g" Makefile
         BUILD_COMMAND ${MAKE}
         BUILD_IN_SOURCE ON
         INSTALL_COMMAND ${MAKE} install
@@ -141,6 +167,13 @@ if(TARGET xz-EXTERNAL)
     endif()
 endif()
 
+# only have to build custom zstd when not using system libzstd
+if(TARGET zstd-EXTERNAL)
+    if(TARGET squashfuse-EXTERNAL)
+        ExternalProject_Add_StepDependencies(squashfuse-EXTERNAL configure zstd-EXTERNAL)
+    endif()
+endif()
+
 ## Boost
 if(NOT USE_SYSTEM_BOOST)
     message(STATUS "Downloading and building boost")

include/appimage/appimage_legacy.h

@@ -10,23 +10,6 @@
  * All of the functions in this header are deprecated and must not be used in newly written code
  */
 
-/*
- * Calculate the size of an ELF file on disk based on the information in its header
- *
- * Example:
- *
- * ls -l   126584
- *
- * Calculation using the values also reported by readelf -h:
- * Start of section headers	e_shoff		124728
- * Size of section headers		e_shentsize	64
- * Number of section headers	e_shnum		29
- *
- * e_shoff + ( e_shentsize * e_shnum ) =	126584
- */
-ssize_t appimage_get_elf_size(const char* fname) __attribute__ ((deprecated));
-
-
 /*
  * Checks whether a type 1 AppImage's desktop file has set Terminal=true.
  *

include/appimage/appimage_shared.h

@@ -38,6 +38,22 @@ char* appimage_hexlify(const char* bytes, size_t numBytes);
  */
 bool appimage_type2_digest_md5(const char* fname, char* digest);
 
+/*
+ * Calculate the size of an ELF file on disk based on the information in its header
+ *
+ * Example:
+ *
+ * ls -l   126584
+ *
+ * Calculation using the values also reported by readelf -h:
+ * Start of section headers	e_shoff		124728
+ * Size of section headers		e_shentsize	64
+ * Number of section headers	e_shnum		29
+ *
+ * e_shoff + ( e_shentsize * e_shnum ) =	126584
+ */
+ssize_t appimage_get_elf_size(const char* fname);
+
 #ifdef __cplusplus
 }
 #endif

src/libappimage/CMakeLists.txt

@@ -33,6 +33,7 @@ foreach(target libappimage libappimage_static)
         # unit tests etc., which use squashfuse directly, must link to it explicitly
         PRIVATE libsquashfuse
         PRIVATE xz
+        PRIVATE zstd
         PRIVATE Boost::filesystem
         PUBLIC libappimage_shared
         PUBLIC pthread

src/libappimage/libappimage_legacy.cpp

@@ -2,10 +2,6 @@
 #include <appimage/appimage.h>
 
 extern "C" {
-ssize_t appimage_get_elf_size(const char* fname) {
-    return appimage_get_payload_offset(fname);
-}
-
 int appimage_type1_is_terminal_app(const char* path) {
     return appimage_is_terminal_app(path);
 } ;

src/libappimage_shared/elf.c

@@ -117,6 +117,41 @@ static off_t read_elf64(FILE* fd)
 	return sht_end > last_section_end ? sht_end : last_section_end;
 }
 
+ssize_t appimage_get_elf_size(const char* fname) {
+    off_t ret;
+    FILE* fd = NULL;
+    off_t size = -1;
+
+	fd = fopen(fname, "rb");
+	if (fd == NULL) {
+		fprintf(stderr, "Cannot open %s: %s\n",
+			fname, strerror(errno));
+		return -1;
+	}
+	ret = fread(ehdr.e_ident, 1, EI_NIDENT, fd);
+	if (ret != EI_NIDENT) {
+		fprintf(stderr, "Read of e_ident from %s failed: %s\n",
+			fname, strerror(errno));
+		return -1;
+	}
+	if ((ehdr.e_ident[EI_DATA] != ELFDATA2LSB) &&
+		(ehdr.e_ident[EI_DATA] != ELFDATA2MSB)) {
+		fprintf(stderr, "Unkown ELF data order %u\n",
+			ehdr.e_ident[EI_DATA]);
+		return -1;
+	}
+	if (ehdr.e_ident[EI_CLASS] == ELFCLASS32) {
+		size = read_elf32(fd);
+	} else if (ehdr.e_ident[EI_CLASS] == ELFCLASS64) {
+		size = read_elf64(fd);
+	} else {
+		fprintf(stderr, "Unknown ELF class %u\n", ehdr.e_ident[EI_CLASS]);
+		return -1;
+	}
+
+	fclose(fd);
+	return size;
+}
 
 /* Return the offset, and the length of an ELF section with a given name in a given ELF file */
 bool appimage_get_elf_section_offset_and_length(const char* fname, const char* section_name, unsigned long* offset, unsigned long* length) {

src/patches/patch-squashfuse.sh.in

@@ -4,5 +4,6 @@ git checkout ll.c Makefile.am fuseprivate.c fuseprivate.h hl.c ll.h ll_inode.c n
 
 patch -p1 < @PROJECT_SOURCE_DIR@/src/patches/squashfuse.patch
 patch -p1 < @PROJECT_SOURCE_DIR@/src/patches/squashfuse_dlopen.patch
+patch -p1 < @PROJECT_SOURCE_DIR@/src/patches/squashfuse_zstd_support.patch
 
 cp -v @PROJECT_SOURCE_DIR@/src/patches/squashfuse_dlopen.c @PROJECT_SOURCE_DIR@/src/patches/squashfuse_dlopen.h .

src/patches/squashfuse_zstd_support.patch

@@ -0,0 +1,112 @@
+From 5986f1d2b002d4f7f7144239a2c39b8aa39ac874 Mon Sep 17 00:00:00 2001
+From: Sean Purcell <[email protected]>
+Date: Mon, 27 Mar 2017 14:42:40 -0700
+Subject: [PATCH] Add zstd compression support
+
+---
+ CONFIGURATION |  1 +
+ Makefile.am   |  4 ++--
+ configure.ac  |  1 +
+ decompress.c  | 21 ++++++++++++++++++++-
+ squashfs_fs.h |  1 +
+ 5 files changed, 25 insertions(+), 3 deletions(-)
+
+diff --git a/CONFIGURATION b/CONFIGURATION
+index c172c3e..044c67a 100644
+--- a/CONFIGURATION
++++ b/CONFIGURATION
+@@ -14,5 +14,6 @@ These are the most useful options to ./configure:
+   --with-xz=PREFIX
+   --with-lzo=PREFIX
+   --with-lz4=PREFIX
++  --with-zstd=PREFIX
+
+ More options are available in `./configure --help'
+diff --git a/Makefile.am b/Makefile.am
+index e84534e..8e61e21 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -1,4 +1,4 @@
+-COMPRESSION_LIBS = $(ZLIB_LIBS) $(XZ_LIBS) $(LZO_LIBS) $(LZ4_LIBS)
++COMPRESSION_LIBS = $(ZLIB_LIBS) $(XZ_LIBS) $(LZO_LIBS) $(LZ4_LIBS) $(ZSTD_LIBS)
+
+ ACLOCAL_AMFLAGS = -I m4 --install
+
+@@ -23,2 +23,2 @@
+ libsquashfuse_la_CPPFLAGS = $(ZLIB_CPPFLAGS) $(XZ_CPPFLAGS) $(LZO_CPPFLAGS) \
+-	$(LZ4_CPPFLAGS)
++	$(LZ4_CPPFLAGS) $(ZSTD_CPPFLAGS)
+diff --git a/configure.ac b/configure.ac
+index 36ffb51..f3b55eb 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -31,6 +31,7 @@ SQ_CHECK_DECOMPRESS([ZLIB],[z],[uncompress],[zlib.h])
+ SQ_CHECK_DECOMPRESS([XZ],[lzma],[lzma_stream_buffer_decode],[lzma.h],[liblzma])
+ SQ_CHECK_DECOMPRESS([LZO],[lzo2],[lzo1x_decompress_safe],[lzo/lzo1x.h])
+ SQ_CHECK_DECOMPRESS([LZ4],[lz4],[LZ4_decompress_safe],[lz4.h])
++SQ_CHECK_DECOMPRESS([ZSTD],[zstd],[ZSTD_decompress],[zstd.h])
+ AS_IF([test "x$sq_decompressors" = x],
+ 	[AC_MSG_FAILURE([At least one decompression library must exist])])
+
+diff --git a/decompress.c b/decompress.c
+index d8a677e..80344f0 100644
+--- a/decompress.c
++++ b/decompress.c
+@@ -98,6 +98,19 @@ static sqfs_err sqfs_decompressor_lz4(void *in, size_t insz,
+ #endif
+
+
++#ifdef HAVE_ZSTD_H
++#include <zstd.h>
++static sqfs_err sqfs_decompressor_zstd(void *in, size_t insz,
++        void *out, size_t *outsz) {
++	const size_t zstdout = ZSTD_decompress(out, *outsz, in, insz);
++	if (ZSTD_isError(zstdout))
++		return SQFS_ERR;
++	*outsz = zstdout;
++	return SQFS_OK;
++}
++#define CAN_DECOMPRESS_ZSTD 1
++#endif
++
+ sqfs_decompressor sqfs_decompressor_get(sqfs_compression_type type) {
+ 	switch (type) {
+ #ifdef CAN_DECOMPRESS_ZLIB
+@@ -111,13 +124,16 @@ sqfs_decompressor sqfs_decompressor_get(sqfs_compression_type type) {
+ #endif
+ #ifdef CAN_DECOMPRESS_LZ4
+ 		case LZ4_COMPRESSION: return &sqfs_decompressor_lz4;
++#endif
++#ifdef CAN_DECOMPRESS_ZSTD
++		case ZSTD_COMPRESSION: return &sqfs_decompressor_zstd;
+ #endif
+ 		default: return NULL;
+ 	}
+ }
+
+ static char *const sqfs_compression_names[SQFS_COMP_MAX] = {
+-	NULL, "zlib", "lzma", "lzo", "xz", "lz4",
++	NULL, "zlib", "lzma", "lzo", "xz", "lz4", "zstd",
+ };
+
+ char *sqfs_compression_name(sqfs_compression_type type) {
+@@ -141,4 +157,7 @@ void sqfs_compression_supported(sqfs_compression_type *types) {
+ #ifdef CAN_DECOMPRESS_LZ4
+ 	types[i++] = LZ4_COMPRESSION;
+ #endif
++#ifdef CAN_DECOMPRESS_ZSTD
++	types[i++] = ZSTD_COMPRESSION;
++#endif
+ }
+diff --git a/squashfs_fs.h b/squashfs_fs.h
+index 7269c1f..e0ab1f4 100644
+--- a/squashfs_fs.h
++++ b/squashfs_fs.h
+@@ -126,6 +126,7 @@
+ #define LZO_COMPRESSION		3
+ #define XZ_COMPRESSION		4
+ #define LZ4_COMPRESSION		5
++#define ZSTD_COMPRESSION	6
+
+ struct squashfs_super_block {
+ 	__le32			s_magic;

tests/libappimage/desktop_integration/CMakeLists.txt

@@ -31,6 +31,7 @@ target_link_libraries(
     PRIVATE libsquashfuse
     PRIVATE libarchive
     PRIVATE xz
+    PRIVATE zstd
     PRIVATE libzlib
     PRIVATE XdgUtils::DesktopEntry
     PRIVATE XdgUtils::BaseDir

tests/libappimage/legacy/test_libappimage.cpp

@@ -303,7 +303,7 @@ bool test_compare_bytes(const char* buf1, const char* buf2, int size) {
 
 TEST_F(LibAppImageTest, appimage_type2_digest_md5) {
     char digest[16];
-    char expectedDigest[] = {-75, -71, 106, -93, 122, 114, 7, 127, -40, 10, -115, -82, -73, 115, -19, 1};
+    char expectedDigest[] = {(char) -75, (char) -71, 106, (char) -93, 122, 114, 7, 127, (char) -40, 10, (char) -115, (char) -82, (char) -73, 115, (char) -19, 1};
 
     EXPECT_TRUE(appimage_type2_digest_md5(appImage_type_2_file_path.c_str(), digest));
     EXPECT_PRED3(test_compare_bytes, digest, expectedDigest, 16);