From fe42bfdab36de28efc285cebfece19c47e0e1a81 Mon Sep 17 00:00:00 2001 From: John Lee Date: Tue, 9 Jul 2024 15:03:32 -0400 Subject: [PATCH 01/12] Add special casing so that mac arm64 defaults to universal builds --- Makefile | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7ad7f219f8..3ace975c5c 100644 --- a/Makefile +++ b/Makefile @@ -174,6 +174,11 @@ ifeq ($(OS_TYPE),darwin) mkdir -p $(GOPATH1)/bin-darwin-arm64 CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=12.0" --host=aarch64-apple-darwin' $(MAKE) + # same for buildsrc-special + cd tools/block-generator && \ + CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOPATH1)/bin-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=12.0" --host=x86_64-apple-darwin' $(MAKE) + CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=12.0" --host=aarch64-apple-darwin' $(MAKE) + # lipo together mkdir -p $(GOPATH1)/bin-darwin-universal for binary in $$(ls $(GOPATH1)/bin-darwin-arm64); do \ @@ -186,7 +191,7 @@ ifeq ($(OS_TYPE),darwin) fi \ done else - $(error OS_TYPE must be darwin for universal builds) + echo "OS_TYPE must be darwin for universal builds, skipping" endif deps: @@ -244,11 +249,45 @@ ${GOCACHE}/file.txt: touch "${GOCACHE}"/file.txt buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN ${GOCACHE}/file.txt +ifeq ($(OS_TYPE),darwin) +ifeq ($(ARCH),arm64) +ifeq (,$(CROSS_COMPILE_ARCH)) + # If we're on an M1 Mac, we need to build the universal binaries + # We should only do this if we're not already cross-compiling + $(MAKE) universal +else + # cross-compiling darwin arm64 + $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... +endif +else + # cross-compiling darwin amd64 or just building on mac amd64 + $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... +endif +else + # everything else like linux $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... +endif buildsrc-special: +ifeq ($(OS_TYPE),darwin) +ifeq ($(ARCH),arm64) +ifeq (,$(CROSS_COMPILE_ARCH)) + echo "Covered under universal target" +else + # cross-compiling darwin arm64 + cd tools/block-generator && \ + $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... +endif +else + # cross-compiling darwin amd64 or just building on mac amd64 + cd tools/block-generator && \ + $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... +endif +else + # everything else like linux cd tools/block-generator && \ $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... +endif check-go-version: ./scripts/check_golang_version.sh build From cea2165dd9cb826b141550e5cb5f967f44ea5576 Mon Sep 17 00:00:00 2001 From: John Lee Date: Wed, 10 Jul 2024 15:03:38 -0400 Subject: [PATCH 02/12] Revert forcing universal build for mac arm64. We should handle this separately --- Makefile | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/Makefile b/Makefile index 3ace975c5c..89588d19d0 100644 --- a/Makefile +++ b/Makefile @@ -249,45 +249,7 @@ ${GOCACHE}/file.txt: touch "${GOCACHE}"/file.txt buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN ${GOCACHE}/file.txt -ifeq ($(OS_TYPE),darwin) -ifeq ($(ARCH),arm64) -ifeq (,$(CROSS_COMPILE_ARCH)) - # If we're on an M1 Mac, we need to build the universal binaries - # We should only do this if we're not already cross-compiling - $(MAKE) universal -else - # cross-compiling darwin arm64 - $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... -endif -else - # cross-compiling darwin amd64 or just building on mac amd64 - $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... -endif -else - # everything else like linux $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... -endif - -buildsrc-special: -ifeq ($(OS_TYPE),darwin) -ifeq ($(ARCH),arm64) -ifeq (,$(CROSS_COMPILE_ARCH)) - echo "Covered under universal target" -else - # cross-compiling darwin arm64 - cd tools/block-generator && \ - $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... -endif -else - # cross-compiling darwin amd64 or just building on mac amd64 - cd tools/block-generator && \ - $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... -endif -else - # everything else like linux - cd tools/block-generator && \ - $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... -endif check-go-version: ./scripts/check_golang_version.sh build From 85c27e0351f5542d2d970e817b551fdcea4f3357 Mon Sep 17 00:00:00 2001 From: John Lee Date: Wed, 10 Jul 2024 15:10:01 -0400 Subject: [PATCH 03/12] Restore missing buildsrc-special target --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 89588d19d0..b3462fbaaa 100644 --- a/Makefile +++ b/Makefile @@ -251,6 +251,10 @@ ${GOCACHE}/file.txt: buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN ${GOCACHE}/file.txt $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... +buildsrc-special: + cd tools/block-generator && \ + $(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./... + check-go-version: ./scripts/check_golang_version.sh build From 9ebeb67a1fd3f534af3c7998e6160864537e3c30 Mon Sep 17 00:00:00 2001 From: John Lee Date: Wed, 10 Jul 2024 16:19:54 -0400 Subject: [PATCH 04/12] Push universal binaries to main bin dir --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b3462fbaaa..faf3fee34a 100644 --- a/Makefile +++ b/Makefile @@ -180,10 +180,10 @@ ifeq ($(OS_TYPE),darwin) CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=12.0" --host=aarch64-apple-darwin' $(MAKE) # lipo together - mkdir -p $(GOPATH1)/bin-darwin-universal + mkdir -p $(GOPATH1)/bin for binary in $$(ls $(GOPATH1)/bin-darwin-arm64); do \ if [ -f $(GOPATH1)/bin-darwin-amd64/$$binary ]; then \ - lipo -create -output $(GOPATH1)/bin-darwin-universal/$$binary \ + lipo -create -output $(GOPATH1)/bin/$$binary \ $(GOPATH1)/bin-darwin-arm64/$$binary \ $(GOPATH1)/bin-darwin-amd64/$$binary; \ else \ From d4d42c69f15c7b4d478704d85bb13023eb6a52dc Mon Sep 17 00:00:00 2001 From: John Lee Date: Mon, 15 Jul 2024 10:31:06 -0400 Subject: [PATCH 05/12] Update to add nightly build option for overriding darwin builds --- .circleci/config.yml | 2 +- scripts/travis/deploy_packages.sh | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4057049dac..13d4ea996f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -715,7 +715,7 @@ commands: command: | if [ "${CIRCLE_BRANCH}" = "rel/nightly" ] then - export NO_BUILD="true" + export NIGHTLY_BUILD="true" fi export PATH=$(echo "$PATH" | sed -e "s|:${HOME}/\.go_workspace/bin||g" | sed -e 's|:/usr/local/go/bin||g') export GOPATH="<< parameters.build_dir >>/go" diff --git a/scripts/travis/deploy_packages.sh b/scripts/travis/deploy_packages.sh index e7e517394c..5879835b01 100755 --- a/scripts/travis/deploy_packages.sh +++ b/scripts/travis/deploy_packages.sh @@ -11,7 +11,7 @@ set -e SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" - +OSARCH=$("${SCRIPTPATH}/../osarchtype.sh") # Get the go build version. GOLANG_VERSION=$("${SCRIPTPATH}/../get_golang_version.sh") @@ -24,6 +24,15 @@ then exit 1 fi +if [ "${NIGHTLY_BUILD}" == "true" ]; then + # we want to rebuild universal binaries for nightly builds + NO_BUILD=true + if [ "${OSARCH}" == "darwin/arm64" ]; then + make universal + OSARCH="darwin/universal" + fi +fi + if [ -z "${NO_BUILD}" ] || [ "${NO_BUILD}" != "true" ]; then scripts/travis/build.sh fi @@ -31,4 +40,4 @@ fi export RELEASE_GENESIS_PROCESS=true export NO_BUILD=true export SkipCleanCheck=1 -scripts/deploy_version.sh "${TRAVIS_BRANCH}" "$(./scripts/osarchtype.sh)" +scripts/deploy_version.sh "${TRAVIS_BRANCH}" "${OSARCH}" From d65d8a3c4955ce9fba2370f8ac2a231f7136b1b4 Mon Sep 17 00:00:00 2001 From: John Lee Date: Mon, 15 Jul 2024 14:54:11 -0400 Subject: [PATCH 06/12] Look for universal versions of binaries if under Darwin --- util/s3/s3Helper.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/util/s3/s3Helper.go b/util/s3/s3Helper.go index efad67d924..7bdc1fb863 100644 --- a/util/s3/s3Helper.go +++ b/util/s3/s3Helper.go @@ -189,7 +189,17 @@ func (helper *Helper) GetPackageVersion(channel string, pkg string, specificVers osName := runtime.GOOS arch := runtime.GOARCH prefix := fmt.Sprintf("%s_%s_%s-%s_", pkg, channel, osName, arch) - return helper.GetPackageFilesVersion(channel, prefix, specificVersion) + + maxVersion, maxVersionName, err = helper.GetPackageFilesVersion(channel, prefix, specificVersion) + // For darwin, we want to also look at universal binaries + if osName == "darwin" { + universalPrefix := fmt.Sprintf("%s_%s_%s-%s_", pkg, channel, osName, "universal") + universalMaxVersion, universalMaxVersionName, universalErr := helper.GetPackageFilesVersion(channel, universalPrefix, specificVersion) + if universalMaxVersion > maxVersion { + return universalMaxVersion, universalMaxVersionName, universalErr + } + } + return } // GetPackageFilesVersion return the package version From a3471267fd33d7e67a6b0e585767e355cca19364 Mon Sep 17 00:00:00 2001 From: John Lee Date: Mon, 15 Jul 2024 14:57:36 -0400 Subject: [PATCH 07/12] Remove arm32 from updater script --- cmd/updater/update.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/updater/update.sh b/cmd/updater/update.sh index 6c20d55c2d..c7f049eec9 100755 --- a/cmd/updater/update.sh +++ b/cmd/updater/update.sh @@ -185,10 +185,6 @@ function get_updater_url() { UNAME=$(uname -m) if [[ "${UNAME}" = "x86_64" ]]; then ARCH="amd64" - elif [[ "${UNAME}" = "armv6l" ]]; then - ARCH="arm" - elif [[ "${UNAME}" = "armv7l" ]]; then - ARCH="arm" elif [[ "${UNAME}" = "aarch64" ]]; then ARCH="arm64" else From cf34ab947b344014ffcc1331b0a9969779217a92 Mon Sep 17 00:00:00 2001 From: John Lee Date: Wed, 17 Jul 2024 15:22:15 -0400 Subject: [PATCH 08/12] Add ci-build-universal target --- scripts/release/mule/Makefile.mule | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/scripts/release/mule/Makefile.mule b/scripts/release/mule/Makefile.mule index 0c73cccf89..06fe5d09b7 100644 --- a/scripts/release/mule/Makefile.mule +++ b/scripts/release/mule/Makefile.mule @@ -1,15 +1,13 @@ # This file is imported into go-algorand/Makefile. PKG_DIR = $(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH) +PKG_DIR_UNIVERSAL = $(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/universal -.PHONY: ci-clean ci-setup ci-build +.PHONY: ci-clean ci-build ci-clean: clean rm -rf tmp -ci-setup: - mkdir -p $(PKG_DIR) - ci-test: ifeq ($(ARCH), amd64) RACE=-race @@ -28,7 +26,16 @@ ci-integration: SRCROOT=$(SRCPATH) \ test/scripts/e2e.sh -c $(CHANNEL) -n -ci-build: ci-clean build ci-setup +ci-build-universal: ci-clean universal + echo $(PKG_DIR_UNIVERSAL) + mkdir -p $(PKG_DIR_UNIVERSAL) + CHANNEL=$(CHANNEL) PKG_ROOT=$(PKG_DIR_UNIVERSAL) NO_BUILD=True VARIATIONS=$(OS_TYPE)-universal \ + scripts/build_packages.sh $(OS_TYPE)/universal && \ + mkdir -p $(PKG_DIR_UNIVERSAL)/data && \ + cp installer/genesis/devnet/genesis.json $(PKG_DIR_UNIVERSAL)/data + +ci-build: ci-clean + mkdir -p $(PKG_DIR) CHANNEL=$(CHANNEL) PKG_ROOT=$(PKG_DIR) NO_BUILD=True VARIATIONS=$(OS_TYPE)-$(ARCH) \ scripts/build_packages.sh $(OS_TYPE)/$(ARCH) && \ mkdir -p $(PKG_DIR)/data && \ From 466224035b5c50561f163114acaea95d1b1fe666 Mon Sep 17 00:00:00 2001 From: John Lee Date: Wed, 17 Jul 2024 15:54:21 -0400 Subject: [PATCH 09/12] package-upload.yaml was deprecated, so removing altogether --- package-upload.yaml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 package-upload.yaml diff --git a/package-upload.yaml b/package-upload.yaml deleted file mode 100644 index 9d13c458ef..0000000000 --- a/package-upload.yaml +++ /dev/null @@ -1,23 +0,0 @@ -tasks: - - task: s3.BucketCopy - name: amd64 - src: $HOME/projects/go-algorand/tmp/node_pkgs/linux/amd64 - dest: s3://$STAGING/$CHANNEL/$VERSION/ - - - task: s3.BucketCopy - name: arm - src: $HOME/projects/go-algorand/tmp/node_pkgs/linux/arm - dest: s3://$STAGING/$CHANNEL/$VERSION/ - - - task: s3.BucketCopy - name: arm64 - src: $HOME/projects/go-algorand/tmp/node_pkgs/linux/arm64 - dest: s3://$STAGING/$CHANNEL/$VERSION/ - -jobs: - package-upload: - tasks: - - s3.BucketCopy.amd64 - - s3.BucketCopy.arm - - s3.BucketCopy.arm64 - From 1f258159a4106a95c49911caee8945abb9a238e3 Mon Sep 17 00:00:00 2001 From: John Lee Date: Wed, 17 Jul 2024 15:57:56 -0400 Subject: [PATCH 10/12] Remove package-upload information which is deprecated --- scripts/release/mule/README.md | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/scripts/release/mule/README.md b/scripts/release/mule/README.md index 8439767725..030728ef70 100644 --- a/scripts/release/mule/README.md +++ b/scripts/release/mule/README.md @@ -25,7 +25,6 @@ In addition, make sure that the following AWS credentials are set in environment # Build Stages - [package](#package) -- [upload](#upload) - [test](#test) - [sign](#sign) - [deploy](#deploy) @@ -49,20 +48,6 @@ In addition, make sure that the following AWS credentials are set in environment - package-docker + packages docker image -## upload - -- see `./go-algorand/package-upload.yaml` - -- customizable environment variables: - - + `CHANNEL` - + `STAGING` - + `VERSION` - -#### `mule` jobs - - - package-upload - ## test - see `./go-algorand/package-test.yaml` @@ -180,10 +165,6 @@ Let's look at some examples. mule -f package.yaml package -### Uploading - - STAGING=the-staging-area CHANNEL=beta VERSION=latest mule -f package-upload.yaml package-upload - ### Testing 1. As part of the test suite, the `verify_package_string.sh` test needs the `BRANCH` as well as the `SHA`: From c72ad64049ddaeb6f2f4880e72d3d6eaa5f3c6ef Mon Sep 17 00:00:00 2001 From: John Lee Date: Wed, 17 Jul 2024 15:58:34 -0400 Subject: [PATCH 11/12] Update darwin targets in CI to point to universal target --- test/muleCI/mule.yaml | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/test/muleCI/mule.yaml b/test/muleCI/mule.yaml index afa73ae2c1..5022cba792 100644 --- a/test/muleCI/mule.yaml +++ b/test/muleCI/mule.yaml @@ -93,11 +93,8 @@ agents: tasks: - task: shell.Make - name: build.darwin-arm64 - target: ci-build - - task: shell.Make - name: build.darwin-amd64 - target: ci-build + name: build.darwin-universal + target: ci-build-universal - task: docker.Make name: build.amd64 agent: cicd.ubuntu.amd64 @@ -124,9 +121,9 @@ tasks: # Stash tasks - task: stash.Stash - name: darwin-arm64 + name: darwin-universal bucketName: go-algorand-ci-cache - stashId: ${JENKINS_JOB_CACHE_ID}/darwin-arm64 + stashId: ${JENKINS_JOB_CACHE_ID}/darwin-universal globSpecs: - tmp/node_pkgs/**/* - task: stash.Stash @@ -135,12 +132,6 @@ tasks: stashId: ${JENKINS_JOB_CACHE_ID}/linux-amd64 globSpecs: - tmp/node_pkgs/**/* - - task: stash.Stash - name: darwin-amd64 - bucketName: go-algorand-ci-cache - stashId: ${JENKINS_JOB_CACHE_ID}/darwin-amd64 - globSpecs: - - tmp/node_pkgs/**/* - task: stash.Stash name: linux-arm64 bucketName: go-algorand-ci-cache @@ -164,13 +155,9 @@ tasks: bucketName: go-algorand-ci-cache stashId: ${JENKINS_JOB_CACHE_ID}/linux-amd64 - task: stash.Unstash - name: darwin-amd64 + name: darwin-universal bucketName: go-algorand-ci-cache - stashId: ${JENKINS_JOB_CACHE_ID}/darwin-amd64 - - task: stash.Unstash - name: darwin-arm64 - bucketName: go-algorand-ci-cache - stashId: ${JENKINS_JOB_CACHE_ID}/darwin-arm64 + stashId: ${JENKINS_JOB_CACHE_ID}/darwin-universal - task: stash.Unstash name: packages bucketName: go-algorand-ci-cache @@ -187,14 +174,10 @@ tasks: target: mule-sign jobs: - build-darwin-arm64: - tasks: - - shell.Make.build.darwin-arm64 - - stash.Stash.darwin-arm64 - build-darwin-amd64: + build-darwin-universal: tasks: - - shell.Make.build.darwin-amd64 - - stash.Stash.darwin-amd64 + - shell.Make.build.darwin-universal + - stash.Stash.darwin-universal build-linux-amd64: tasks: - docker.Make.build.amd64 @@ -207,8 +190,7 @@ jobs: tasks: - stash.Unstash.linux-amd64 - stash.Unstash.linux-arm64 - - stash.Unstash.darwin-arm64 - - stash.Unstash.darwin-amd64 + - stash.Unstash.darwin-universal - docker.Make.deb.amd64 - docker.Make.rpm.amd64 - stash.Stash.packages From ebb1b5c83b513e97876a3365a9b177abb9b9bdac Mon Sep 17 00:00:00 2001 From: John Lee Date: Wed, 24 Jul 2024 10:41:50 -0400 Subject: [PATCH 12/12] Remove naked return so it's clearer Co-authored-by: Pavel Zbitskiy <65323360+algorandskiy@users.noreply.github.com> --- util/s3/s3Helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/s3/s3Helper.go b/util/s3/s3Helper.go index 7bdc1fb863..79396e7ced 100644 --- a/util/s3/s3Helper.go +++ b/util/s3/s3Helper.go @@ -199,7 +199,7 @@ func (helper *Helper) GetPackageVersion(channel string, pkg string, specificVers return universalMaxVersion, universalMaxVersionName, universalErr } } - return + return maxVersion, maxVersionName, err } // GetPackageFilesVersion return the package version