make: factorize code for generating release updates

Instead of splitting remove / exclusion of unwanted files in 2-3 places (`all` rule, `update` rule, and update script in some cases), install the same files in debug & release builds, and exclude unwanted stuff when generating the update. This should finally fix the CI issue with OCR tests (since we don't remove the `data/dict` & `data/tessdata` directories on release build anymore).
benoit-pierre · Nov 19, 2024 · 9fdaa33 · 9fdaa33
1 parent bcb7753
commit 9fdaa33
Show file tree

Hide file tree

Showing 15 changed files with 544 additions and 345 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -197,7 +197,9 @@ jobs:
       # Generate / upload artifact. {{{
 
       - name: Generate artifact
-        run: make update --assume-old=base
+        # Note: ensure the build skeleton is created, so there are no dangling
+        # symlinks (e.g. `data/dict`) to trip 7zip during artifact generation…
+        run: make skeleton update --assume-old=base
 
       - name: Upload artifact
         uses: actions/upload-artifact@v4

diff --git a/Makefile b/Makefile
@@ -14,10 +14,6 @@ ifneq (,$(findstring -,$(VERSION)))
 	VERSION := $(VERSION)_$(RELEASE_DATE)
 endif
 
-# releases do not contain tests and misc data
-IS_RELEASE := $(if $(or $(EMULATE_READER),$(WIN32)),,1)
-IS_RELEASE := $(if $(or $(IS_RELEASE),$(APPIMAGE),$(LINUX),$(MACOS)),1,)
-
 LINUX_ARCH?=native
 ifeq ($(LINUX_ARCH), native)
 	LINUX_ARCH_NAME:=$(shell uname -m)
@@ -50,11 +46,14 @@ WIN32_DIR=$(PLATFORM_DIR)/win32
 
 define CR3GUI_DATADIR_EXCLUDES
 %/KoboUSBMS.tar.gz
+%/NOTES.txt
 %/cr3.ini
 %/cr3skin-format.txt
 %/desktop
 %/devices
+%/dict
 %/manual
+%/tessdata
 endef
 CR3GUI_DATADIR_FILES = $(filter-out $(CR3GUI_DATADIR_EXCLUDES),$(wildcard $(CR3GUI_DATADIR)/*))
 
@@ -71,18 +70,78 @@ INSTALL_FILES=reader.lua setupkoenv.lua frontend resources defaults.lua datastor
 OUTPUT_DIR_ARTIFACTS = $(abspath $(OUTPUT_DIR))/!(cache|cmake|data|history|staging|thirdparty)
 OUTPUT_DIR_DATAFILES = $(OUTPUT_DIR)/data/*
 
+# Release excludes. {{{
+
+define UPDATE_PATH_EXCLUDES
+cache
+clipboard
+data/dict
+data/tessdata
+ev_replay.py
+help
+history
+l10n/templates
+ota
+resources/fonts*
+resources/icons/src*
+screenshots
+spec
+endef
+
+# Files created after execution.
+define UPDATE_PATH_EXCLUDES +=
+data/cr3.ini
+defaults.*.lua
+history.lua
+scripts
+settings
+settings.reader.lua*
+styletweaks
+version.log
+endef
+
+# Testsuite leftovers.
+define UPDATE_PATH_EXCLUDES +=
+dummy-test-file*
+file.sdr*
+readerbookmark.*
+readerhighlight.*
+testdata
+this-is-not-a-valid-file*
+endef
+
+# Globally excluded.
+define UPDATE_GLOBAL_EXCLUDES
+*.orig
+*.swo
+*.swp
+*.un~
+.*
+endef
+
+release_excludes = $(strip $(UPDATE_PATH_EXCLUDES:%='-x!$1%') $(UPDATE_GLOBAL_EXCLUDES:%='-xr!%'))
+
+# }}}
+
+define mkupdate
+cd $(INSTALL_DIR) &&
+'$(abspath tools/mkrelease.sh)'
+$(if $(PARALLEL_JOBS),--jobs $(PARALLEL_JOBS))
+--manifest=$(or $2,koreader)/ota/package.index
+$(foreach a,$1,'$(if $(filter --%,$a),$a,$(abspath $a))') $(or $2,koreader)
+$(call release_excludes,$(or $2,koreader)/)
+endef
+
 all: base
 	install -d $(INSTALL_DIR)/koreader
 	rm -f $(INSTALL_DIR)/koreader/git-rev; echo "$(VERSION)" > $(INSTALL_DIR)/koreader/git-rev
 ifdef ANDROID
 	rm -f android-fdroid-version; echo -e "$(ANDROID_NAME)\n$(ANDROID_VERSION)" > koreader-android-fdroid-latest
 endif
-ifeq (,$(IS_RELEASE))
 	$(SYMLINK) $(KOR_BASE)/ev_replay.py $(INSTALL_DIR)/koreader/
-endif
 	bash -O extglob -c '$(SYMLINK) $(OUTPUT_DIR_ARTIFACTS) $(INSTALL_DIR)/koreader/'
 ifneq (,$(EMULATE_READER))
-	@echo "[*] install front spec only for the emulator"
+	@echo "[*] Install front spec only for the emulator"
 	$(SYMLINK) spec $(INSTALL_DIR)/koreader/spec/front
 	$(SYMLINK) test $(INSTALL_DIR)/koreader/spec/front/unit/data
 endif
@@ -112,10 +171,6 @@ endif
 	! test -L $(INSTALL_DIR)/koreader/data || rm $(INSTALL_DIR)/koreader/data
 	install -d $(INSTALL_DIR)/koreader/data
 	$(SYMLINK) $(strip $(DATADIR_FILES)) $(INSTALL_DIR)/koreader/data/
-ifneq (,$(IS_RELEASE))
-	@echo "[*] Clean up, remove unused files for releases"
-	rm -rf $(INSTALL_DIR)/koreader/data/{cr3.ini,desktop,devices,dict,manual,tessdata}
-endif
 
 base: base-all
 
@@ -198,3 +253,5 @@ doc:
 .PHONY: $(PHONY)
 
 include $(KOR_BASE)/Makefile
+
+# vim: foldmethod=marker foldlevel=0
diff --git a/make/android.mk b/make/android.mk
@@ -42,6 +42,12 @@ ANDROID_DIR = $(PLATFORM_DIR)/android
 ANDROID_LAUNCHER_DIR = $(ANDROID_DIR)/luajit-launcher
 ANDROID_LAUNCHER_BUILD = $(INSTALL_DIR)/luajit-launcher
 ANDROID_ASSETS = $(ANDROID_LAUNCHER_BUILD)/assets
+# Assets compression method:
+# - LZMA/LZMA2: `-m0=lzma2 -mx=9`
+# - LZMA/LZMA2 (7z >= 17.02, fast version): `-m0=flzma2 -mx=9`
+# - ZSTD (7z >= 17.02, LZMA still used for archive headers): `-m0=zstd -mx=16`
+# - ZSTD (7z >= 17.02): `-m0=zstd -mhc=off -mx=16`
+ANDROID_ASSETS_COMPRESSION ?= -m0=lzma2 -mx=9
 ANDROID_LIBS = $(ANDROID_LAUNCHER_BUILD)/libs/$(ANDROID_ABI)
 ANDROID_FLAVOR ?= Rocks
 
@@ -61,6 +67,22 @@ else
   ANDROID_ABI ?= armeabi-v7a
 endif
 
+define UPDATE_PATH_EXCLUDES +=
+plugins/SSH.koplugin
+plugins/autofrontlight.koplugin
+plugins/hello.koplugin
+plugins/timesync.koplugin
+tools
+endef
+
+define UPDATE_GLOBAL_EXCLUDES +=
+README.md
+COPYING
+LICENSE*
+*license.txt
+NOTICE
+endef
+
 androiddev: update
 	$(MAKE) -C $(ANDROID_LAUNCHER_DIR) dev
 
@@ -83,35 +105,14 @@ update: all
 	# binaries are stored as shared libraries to prevent W^X exception on Android 10+
 	# https://developer.android.com/about/versions/10/behavior-changes-10#execute-permission
 	llvm-strip --strip-unneeded $(INSTALL_DIR)/koreader/sdcv -o $(ANDROID_LIBS)/libsdcv.so
-	# assets are compressed manually and stored inside the APK.
+	# Assets are compressed manually and stored inside the APK.
 	cd $(INSTALL_DIR)/koreader && \
-		./tools/mk7z.sh \
-		$(abspath $(ANDROID_ASSETS)/module/koreader.7z) \
-		"$$(git show -s --format='%ci')" \
-		-m0=lzma2 -mx=9 \
-		-- . \
-		'-x!cache' \
-		'-x!clipboard' \
-		'-x!data/dict' \
-		'-x!data/tessdata' \
-		'-x!history' \
-		'-x!l10n/templates' \
-		'-x!libs' \
-		'-x!ota' \
-		'-x!resources/fonts*' \
-		'-x!resources/icons/src*' \
-		'-x!rocks/bin' \
-		'-x!rocks/lib/luarocks' \
-		'-x!screenshots' \
-		'-x!sdcv' \
-		'-x!spec' \
-		'-x!tools' \
-		'-xr!.*' \
-		'-xr!COPYING' \
-		'-xr!NOTES.txt' \
-		'-xr!NOTICE' \
-		'-xr!README.md' \
-		;
+	  ./tools/mkrelease.sh \
+	  $(if $(PARALLEL_JOBS),--jobs $(PARALLEL_JOBS)) \
+	  --epoch="$$(git show -s --format='%ci')" \
+	  --options='$(ANDROID_ASSETS_COMPRESSION)' \
+	  $(abspath $(ANDROID_ASSETS)/module/koreader.7z) \
+	  . '-x!libs' '-x!sdcv' $(release_excludes)
 	# Note: we filter out the `--debug=…` make flag so the old
 	# crummy version provided by the NDK does not blow a gasket.
 	env \

diff --git a/make/appimage.mk b/make/appimage.mk
@@ -1,31 +1,36 @@
 APPIMAGE_DIR = $(PLATFORM_DIR)/appimage
 
 APPIMAGETOOL = appimagetool-x86_64.AppImage
-APPIMAGETOOL_URL = https://github.com/AppImage/AppImageKit/releases/download/13/appimagetool-x86_64.AppImage
+APPIMAGETOOL_URL = https://github.com/AppImage/AppImageKit/releases/download/13/$(APPIMAGETOOL)
+
+KOREADER_APPIMAGE = koreader-$(DIST)-$(MACHINE)-$(VERSION).AppImage
+
+UBUNTU_LIBBSD = /lib/x86_64-linux-gnu/libbsd.so.0
+
+define UPDATE_PATH_EXCLUDES +=
+plugins/SSH.koplugin
+plugins/autofrontlight.koplugin
+plugins/timesync.koplugin
+$(filter-out tools/trace_require.lua tools/wbuilder.lua,$(wildcard tools/*))
+endef
 
 update: all
-	# remove old package if any
-	rm -f koreader-appimage-$(MACHINE)-$(VERSION).appimage
-	$(SYMLINK) $(APPIMAGE_DIR)/AppRun $(INSTALL_DIR)/koreader/
-	$(SYMLINK) $(APPIMAGE_DIR)/koreader.desktop $(INSTALL_DIR)/koreader/
-	$(SYMLINK) resources/koreader.png $(INSTALL_DIR)/koreader/
-	sed -e 's/%%VERSION%%/$(VERSION)/' -e 's/%%DATE%%/$(RELEASE_DATE)/' $(PLATFORM_DIR)/common/koreader.metainfo.xml >$(INSTALL_DIR)/koreader/koreader.appdata.xml
-	# also copy libbsd.so.0, cf. https://github.com/koreader/koreader/issues/4627
-	$(SYMLINK) /lib/x86_64-linux-gnu/libbsd.so.0 $(INSTALL_DIR)/koreader/libs/
-ifeq ("$(wildcard $(APPIMAGETOOL))","")
-	# download appimagetool
-	wget "$(APPIMAGETOOL_URL)"
-	chmod a+x "$(APPIMAGETOOL)"
+	cd $(INSTALL_DIR)/koreader && '$(abspath tools/mkrelease.sh)' ../appimage/ . $(release_excludes)
+	cp $(APPIMAGE_DIR)/{AppRun,koreader.desktop} resources/koreader.png $(INSTALL_DIR)/appimage/
+	sed -e 's/%%VERSION%%/$(VERSION)/' -e 's/%%DATE%%/$(RELEASE_DATE)/' $(PLATFORM_DIR)/common/koreader.metainfo.xml >$(INSTALL_DIR)/appimage/koreader.appdata.xml
+	# Also copy libbsd.so.0 (cf. https://github.com/koreader/koreader/issues/4627).
+ifeq (,$(wildcard $(UBUNTU_LIBBSD)))
+	# Only warn if it's missing (e.g. when testing on a non-Ubuntu distribution).
+	echo 'WARNING: not bundling missing $(UBUNTU_LIBBSD)' 1>&2
+else
+	cp $(UBUNTU_LIBBSD) $(INSTALL_DIR)/appimage/libs/
+endif
+ifeq (,$(wildcard $(APPIMAGETOOL)))
+	# Download appimagetool.
+	wget '$(APPIMAGETOOL_URL)'
+	chmod a+x ./$(APPIMAGETOOL)
 endif
-	cd $(INSTALL_DIR) && pwd && \
-		rm -rf tmp && mkdir -p tmp && \
-		cp -Lr koreader tmp && \
-		rm -rf tmp/koreader/ota && \
-		rm -rf tmp/koreader/resources/icons/src && \
-		rm -rf tmp/koreader/spec
-	# generate AppImage
-	cd $(INSTALL_DIR)/tmp && \
-		ARCH=x86_64 "$$OLDPWD/$(APPIMAGETOOL)" --appimage-extract-and-run koreader && \
-		mv *.AppImage ../../koreader-$(DIST)-$(MACHINE)-$(VERSION).AppImage
+	# Generate AppImage.
+	ARCH=x86_64 ./$(APPIMAGETOOL) --appimage-extract-and-run $(INSTALL_DIR)/appimage $(KOREADER_APPIMAGE)
 
 PHONY += update
diff --git a/make/cervantes.mk b/make/cervantes.mk
@@ -2,32 +2,21 @@ CERVANTES_DIR = $(PLATFORM_DIR)/cervantes
 CERVANTES_PACKAGE = koreader-cervantes$(KODEDUG_SUFFIX)-$(VERSION).zip
 CERVANTES_PACKAGE_OTA = koreader-cervantes$(KODEDUG_SUFFIX)-$(VERSION).targz
 
+define UPDATE_PATH_EXCLUDES +=
+tools
+endef
+
 update: all
 	# ensure that the binaries were built for ARM
 	file --dereference $(INSTALL_DIR)/koreader/luajit | grep ARM
 	# remove old package if any
 	rm -f $(CERVANTES_PACKAGE)
 	# Cervantes launching scripts
-	cp $(COMMON_DIR)/spinning_zsync $(INSTALL_DIR)/koreader/spinning_zsync.sh
-	cp $(CERVANTES_DIR)/*.sh $(INSTALL_DIR)/koreader
-	cp $(CERVANTES_DIR)/spinning_zsync $(INSTALL_DIR)/koreader
-	# create new package
-	cd $(INSTALL_DIR) && \
-		zip -9 -r \
-			../$(CERVANTES_PACKAGE) \
-			koreader -x "koreader/resources/fonts/*" \
-			"koreader/resources/icons/src/*" "koreader/spec/*" \
-			$(ZIP_EXCLUDE)
-	# generate update package index file
-	zipinfo -1 $(CERVANTES_PACKAGE) > \
-		$(INSTALL_DIR)/koreader/ota/package.index
-	echo "koreader/ota/package.index" >> $(INSTALL_DIR)/koreader/ota/package.index
-	# update index file in zip package
-	cd $(INSTALL_DIR) && zip -u ../$(CERVANTES_PACKAGE) \
-	koreader/ota/package.index
-	# make gzip cervantes update for zsync OTA update
-	cd $(INSTALL_DIR) && \
-	tar --hard-dereference -I"gzip --rsyncable" -cah --no-recursion -f ../$(CERVANTES_PACKAGE_OTA) \
-	-T koreader/ota/package.index
+	$(SYMLINK) $(COMMON_DIR)/spinning_zsync $(INSTALL_DIR)/koreader/spinning_zsync.sh
+	$(SYMLINK) $(CERVANTES_DIR)/*.sh $(INSTALL_DIR)/koreader
+	$(SYMLINK) $(CERVANTES_DIR)/spinning_zsync $(INSTALL_DIR)/koreader
+	# Create packages.
+	$(strip $(call mkupdate,$(CERVANTES_PACKAGE)))
+	$(strip $(call mkupdate,$(CERVANTES_PACKAGE_OTA)))
 
 PHONY += update
diff --git a/make/kindle.mk b/make/kindle.mk
@@ -1,18 +1,20 @@
 KINDLE_DIR = $(PLATFORM_DIR)/kindle
 KINDLE_PACKAGE = koreader-$(DIST)$(KODEDUG_SUFFIX)-$(VERSION).zip
+# Note: the targz extension is intended to keep ISP from caching the file (Cf. koreader#1644).
 KINDLE_PACKAGE_OTA = koreader-$(DIST)$(KODEDUG_SUFFIX)-$(VERSION).targz
-ZIP_EXCLUDE = -x "*.swp" -x "*.swo" -x "*.orig" -x "*.un~"
 
 # Don't bundle launchpad on touch devices..
 ifeq ($(TARGET), kindle-legacy)
   KINDLE_LEGACY_LAUNCHER = launchpad
 endif
 
+define UPDATE_PATH_EXCLUDES +=
+tools
+endef
+
 update: all
 	# ensure that the binaries were built for ARM
 	file --dereference $(INSTALL_DIR)/koreader/luajit | grep ARM
-	# remove old package if any
-	rm -f $(KINDLE_PACKAGE)
 	# Kindle launching scripts
 	$(SYMLINK) $(KINDLE_DIR)/extensions $(INSTALL_DIR)/
 	$(SYMLINK) $(KINDLE_DIR)/launchpad $(INSTALL_DIR)/
@@ -21,26 +23,8 @@ update: all
 	$(SYMLINK) $(KINDLE_DIR)/libkohelper.sh $(INSTALL_DIR)/extensions/koreader/bin/
 	$(SYMLINK) $(COMMON_DIR)/spinning_zsync $(INSTALL_DIR)/koreader/
 	$(SYMLINK) $(KINDLE_DIR)/wmctrl $(INSTALL_DIR)/koreader/
-	# create new package
-	cd $(INSTALL_DIR) && pwd && \
-		zip -9 -r \
-			../$(KINDLE_PACKAGE) \
-			extensions koreader $(KINDLE_LEGACY_LAUNCHER) \
-			-x "koreader/resources/fonts/*" "koreader/ota/*" \
-			"koreader/resources/icons/src/*" "koreader/spec/*" \
-			$(ZIP_EXCLUDE)
-	# generate kindleupdate package index file
-	zipinfo -1 $(KINDLE_PACKAGE) > \
-		$(INSTALL_DIR)/koreader/ota/package.index
-	echo "koreader/ota/package.index" >> $(INSTALL_DIR)/koreader/ota/package.index
-	# update index file in zip package
-	cd $(INSTALL_DIR) && zip -u ../$(KINDLE_PACKAGE) \
-		koreader/ota/package.index
-	# make gzip kindleupdate for zsync OTA update
-	# note that the targz file extension is intended to keep ISP from caching
-	# the file, see koreader#1644.
-	cd $(INSTALL_DIR) && \
-		tar --hard-dereference -I"gzip --rsyncable" -cah --no-recursion -f ../$(KINDLE_PACKAGE_OTA) \
-		-T koreader/ota/package.index
+	# Create packages.
+	$(strip $(call mkupdate,$(KINDLE_PACKAGE))) extensions $(KINDLE_LEGACY_LAUNCHER)
+	$(strip $(call mkupdate,$(KINDLE_PACKAGE_OTA))) extensions $(KINDLE_LEGACY_LAUNCHER)
 
 PHONY += update