From c50f1a47174fc3e0f2ec2211eb083b416a5d1415 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 31 May 2024 02:32:07 +0100 Subject: [PATCH] phase1: remove kmods in target packages if archive is enabled OPKG gets confused if kmod packages are present in both, target packages as well as kernel version specific folder. Remove them from target packages to make opkg pick the kmods from kmod archive folder only. We reorder the step by: 1) Move the kmods to dedicated directory 2) Index packages 3) Index kmods 4) Make checksums 5) Extract kmods sums from remote sums 6) Merge the local sums with the remote kmods-only sums This permits to produce a better sha256sums that also reflect the hash of the entire directory. Co-developed-by: Christian Marangi Signed-off-by: Daniel Golle [ rework implementation, sha256sumsclean + --remove-source-files ] Signed-off-by: Christian Marangi --- phase1/master.cfg | 167 +++++++++++++++++++++++++++++----------------- 1 file changed, 106 insertions(+), 61 deletions(-) diff --git a/phase1/master.cfg b/phase1/master.cfg index 82ebc90..c066c26 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -588,6 +588,10 @@ def IsKmodArchiveAndRsyncEnabled(step): return bool(IsKmodArchiveEnabled(step) and branches[branch].get("bin_url")) +def IsRemoteShaSumsAvailable(step): + return step.getProperty("have_remote_shasums") + + def GetBaseVersion(branch): if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch): return branch.split("-")[1] @@ -1250,23 +1254,6 @@ def prepareFactory(target): ) ) - factory.addStep( - ShellCommand( - name="pkgindex", - description="Indexing packages", - descriptionDone="Packages indexed", - command=[ - "make", - Interpolate("-j%(prop:nproc:-1)s"), - "package/index", - "V=s", - "CONFIG_SIGNED_PACKAGES=", - ], - env=MakeEnv(), - haltOnFailure=True, - ) - ) - factory.addStep( ShellCommand( name="images", @@ -1303,17 +1290,6 @@ def prepareFactory(target): ) ) - factory.addStep( - ShellCommand( - name="checksums", - description="Calculating checksums", - descriptionDone="Checksums calculated", - command=["make", "-j1", "checksum", "V=s"], - env=MakeEnv(), - haltOnFailure=True, - ) - ) - factory.addStep( ShellCommand( name="kmoddir", @@ -1335,10 +1311,11 @@ def prepareFactory(target): factory.addStep( ShellCommand( name="kmodprepare", - description="Preparing kmod archive", - descriptionDone="Kmod archive prepared", + description="Moving kmod to archive", + descriptionDone="Kmod moved", command=[ "rsync", + "--remove-source-files", "--include=/kmod-*.ipk", "--include=/kmod-*.apk", "--exclude=*", @@ -1359,6 +1336,23 @@ def prepareFactory(target): ) ) + factory.addStep( + ShellCommand( + name="pkgindex", + description="Indexing packages", + descriptionDone="Packages indexed", + command=[ + "make", + Interpolate("-j%(prop:nproc:-1)s"), + "package/index", + "V=s", + "CONFIG_SIGNED_PACKAGES=", + ], + env=MakeEnv(), + haltOnFailure=True, + ) + ) + factory.addStep( ShellCommand( name="kmodindex", @@ -1382,6 +1376,88 @@ def prepareFactory(target): ) ) + factory.addStep( + ShellCommand( + name="checksums", + description="Calculating checksums", + descriptionDone="Checksums calculated", + command=["make", "-j1", "checksum", "V=s"], + env=MakeEnv(), + haltOnFailure=True, + ) + ) + + # download remote sha256sums to 'target-sha256sums' + factory.addStep( + ShellCommandAndSetProperty( + name="target-sha256sums", + description="Fetching remote sha256sums for target", + descriptionDone="Remote sha256sums for target fetched", + command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")] + + rsync_defopts + + [ + Interpolate( + "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", + url=GetRsyncParams.withArgs("bin", "url"), + target=target, + subtarget=subtarget, + prefix=GetVersionPrefix, + ), + "target-sha256sums", + ], + env={ + "RSYNC_PASSWORD": Interpolate( + "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key") + ) + }, + property="have_remote_shasums", + logEnviron=False, + haltOnFailure=False, + flunkOnFailure=False, + warnOnFailure=False, + doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")), + ) + ) + + factory.addStep( + ShellCommand( + name="target-sha256sums_kmodsparse", + description="Extract kmods from remote sha256sums", + descriptionDone="Kmods extracted", + command="sed \"/ \\*kmods\\//! d\" target-sha256sums | tee target-sha256sums-kmods", + haltOnFailure=False, + doStepIf=IsRemoteShaSumsAvailable, + ) + ) + + factory.addStep( + ShellCommand( + name="mergesha256sum", + description="Merge sha256sums kmods with sha256sums", + descriptionDone="Sha256sums merged", + command=[ + "sort", + "-t", " ", + "-k", 2, + "-u", + Interpolate( + "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", + target=target, + subtarget=subtarget, + ), + "target-sha256sums-kmods", + "-o", + Interpolate( + "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", + target=target, + subtarget=subtarget, + ), + ], + haltOnFailure=False, + doStepIf=IsRemoteShaSumsAvailable, + ) + ) + # sign factory.addStep( MasterShellCommand( @@ -1540,37 +1616,6 @@ def prepareFactory(target): ) ) - # download remote sha256sums to 'target-sha256sums' - factory.addStep( - ShellCommand( - name="target-sha256sums", - description="Fetching remote sha256sums for target", - descriptionDone="Remote sha256sums for target fetched", - command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")] - + rsync_defopts - + [ - Interpolate( - "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", - url=GetRsyncParams.withArgs("bin", "url"), - target=target, - subtarget=subtarget, - prefix=GetVersionPrefix, - ), - "target-sha256sums", - ], - env={ - "RSYNC_PASSWORD": Interpolate( - "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key") - ) - }, - logEnviron=False, - haltOnFailure=False, - flunkOnFailure=False, - warnOnFailure=False, - doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")), - ) - ) - # build list of files to upload factory.addStep( FileDownload(