From 3ad4147264cbedcdb9f3b8a9e0a86f927c391f08 Mon Sep 17 00:00:00 2001 From: Jackson West Date: Tue, 10 Oct 2023 22:26:59 -0500 Subject: [PATCH] improves local check by turning into a prereq --- Common.mk | 39 ++++++++++++++++++++------------------- Makefile | 38 ++++++++++++++++++++------------------ 2 files changed, 40 insertions(+), 37 deletions(-) diff --git a/Common.mk b/Common.mk index 1bea2c0c44..a48ade1253 100644 --- a/Common.mk +++ b/Common.mk @@ -437,23 +437,6 @@ BUILD_TARGETS?=github-rate-limit-pre validate-checksums attribution $(if $(IMAGE RELEASE_TARGETS?=validate-checksums $(if $(IMAGE_NAMES),images,) $(if $(filter true,$(HAS_HELM_CHART)),helm/push,) $(if $(filter true,$(HAS_S3_ARTIFACTS)),upload-artifacts,) #################################################### -# Locale settings impact file ordering in ls or shell file expansion. The file order is used to -# generate files that are subsequently validated by the CI. If local environments use different -# locales to the CI we get unexpected failures that are tricky to debug without knowledge of -# locales so we'll explicitly warn here. -# In a AL2 container image (like builder base), LANG will be empty which is equilvant to posix -# In a AL2 (or other distro) full instance the LANG will be en-us.UTF-8 which produces different sorts -# On Mac, LANG will be en-us.UTF-8 but has a fix applied to sort to avoid the difference -ifeq ($(shell uname -s),Linux) - LOCALE:=$(call TO_LOWER,$(shell locale | grep LANG | cut -d= -f2 | tr -d '"')) - LOCALE:=$(if $(LOCALE),$(LOCALE),posix) - ifeq ($(filter c.utf-8 posix,$(LOCALE)),) - $(warning WARNING: Environment locale set to $(LANG). On Linux systems this may create \ - non-deterministic behavior when running generation recipes. If the CI fails validation try \ - `LANG=C.UTF-8 make ` to generate files instead.) - endif -endif - define BUILDCTL $(BUILD_LIB)/buildkit.sh \ build \ @@ -621,7 +604,7 @@ $(OUTPUT_DIR)/%TTRIBUTION.txt: $(OUTPUT_DIR)/%ttribution/go-license.csv: BINARY_TARGET=$(if $(filter .,$(*D)),,$(*D)) $(OUTPUT_DIR)/%ttribution/go-license.csv: GO_MOD_PATH=$(if $(BINARY_TARGET),$(GO_MOD_TARGET_FOR_BINARY_$(call TO_UPPER,$(BINARY_TARGET))),$(word 1,$(UNIQ_GO_MOD_PATHS))) $(OUTPUT_DIR)/%ttribution/go-license.csv: LICENSE_PACKAGE_FILTER=$(GO_MOD_$(subst /,_,$(GO_MOD_PATH))_LICENSE_PACKAGE_FILTER) -$(OUTPUT_DIR)/%ttribution/go-license.csv: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH)) +$(OUTPUT_DIR)/%ttribution/go-license.csv: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH)) @echo -e $(call TARGET_START_LOG) $(BASE_DIRECTORY)/build/lib/gather_licenses.sh $(REPO) $(MAKE_ROOT)/$(OUTPUT_DIR)/$(BINARY_TARGET) "$(LICENSE_PACKAGE_FILTER)" $(GO_MOD_PATH) $(GOLANG_VERSION) $(LICENSE_THRESHOLD) @echo -e $(call TARGET_END_LOG) @@ -913,7 +896,7 @@ stop-docker-builder: # Clean up builder base docker container docker rm -f -v eks-a-builder .PHONY: generate -generate: # Update UPSTREAM_PROJECTS.yaml +generate: | ensure-locale # Update UPSTREAM_PROJECTS.yaml $(BUILD_LIB)/generate_projects_list.sh $(BASE_DIRECTORY) .PHONY: update-go-mods @@ -988,6 +971,24 @@ github-rate-limit-%: GH_PAGER='' gh api rate_limit; \ fi +# Locale settings impact file ordering in ls or shell file expansion. The file order is used to +# generate files that are subsequently validated by the CI. If local environments use different +# locales to the CI we get unexpected failures that are tricky to debug without knowledge of +# locales so we'll explicitly warn here. +# In a AL2 container image (like builder base), LANG will be empty which is equivalent to posix +# In a AL2 (or other distro) full instance the LANG will be en-us.UTF-8 which produces different sorts +# On Mac, LANG will be en-us.UTF-8 but has a fix applied to sort to avoid the difference +.PHONY: ensure-locale +ensure-locale: + @if [ "Linux" = "$$(uname -s)" ]; then \ + LOCALE=$$(locale | grep LANG | cut -d= -f2 | tr -d '"' | tr '[:upper:]' '[:lower:]'); \ + if [[ "c.utf-8 posix" != *"$${LOCALE:-posix}"* ]]; then \ + echo WARNING: Environment locale set to $(LANG). On Linux systems this may create \ + non-deterministic behavior when running generation recipes. If the CI fails validation try \ + exporting LANG=C.UTF-8 to generate files instead.; \ + fi; \ + fi + ## -------------------------------------- ## Docker Helpers ## -------------------------------------- diff --git a/Makefile b/Makefile index 02f6046625..61eb660383 100644 --- a/Makefile +++ b/Makefile @@ -13,22 +13,6 @@ ALL_PROJECTS=$(shell $(BUILD_LIB)/all_projects.sh $(BASE_DIRECTORY)) # $1 - project name using _ as separator, ex: rancher_local-path-provisoner PROJECT_PATH_MAP=projects/$(patsubst $(firstword $(subst _, ,$(1)))_%,$(firstword $(subst _, ,$(1)))/%,$(1)) -# Locale settings impact file ordering in ls or shell file expansion. The file order is used to -# generate files that are subsequently validated by the CI. If local environments use different -# locales to the CI we get unexpected failures that are tricky to debug without knowledge of -# locales so we'll explicitly warn here. -# In a AL2 container image (like builder base), LANG will be empty which is equivalent to posix -# In a AL2 (or other distro) full instance the LANG will be en-us.UTF-8 which produces different sorts -# On Mac, LANG will be en-us.UTF-8 but has a fix applied to sort to avoid the difference -ifeq ($(shell uname -s),Linux) - LOCALE:=$(call TO_LOWER,$(shell locale | grep LANG | cut -d= -f2 | tr -d '"')) - LOCALE:=$(if $(LOCALE),$(LOCALE),posix) - ifeq ($(filter c.utf-8 posix,$(LOCALE)),) - $(warning WARNING: Environment locale set to $(LANG). On Linux systems this may create \ - non-deterministic behavior when running generation recipes. If the CI fails validation try \ - `LANG=C.UTF-8 make ` to generate files instead.) - endif -endif .PHONY: clean-project-% clean-project-%: @@ -86,11 +70,11 @@ stop-buildkit-and-registry: docker rm -v --force registry .PHONY: generate-project-list -generate-project-list: +generate-project-list: | ensure-locale build/lib/generate_projects_list.sh $(BASE_DIRECTORY) .PHONY: generate-staging-buildspec -generate-staging-buildspec: +generate-staging-buildspec: | ensure-locale build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/release/staging-build.yml" build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/release/checksums-build.yml" true EXCLUDE_FROM_CHECKSUMS_BUILDSPEC CHECKSUMS_BUILDSPECS false buildspecs/checksums-pr-buildspec.yml build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "aws_bottlerocket-bootstrap" "$(BASE_DIRECTORY)/projects/aws/bottlerocket-bootstrap/buildspecs/batch-build.yml" true @@ -122,3 +106,21 @@ check-project-path-exists: .PHONY: validate-eksd-releases validate-eksd-releases: build/lib/validate_eksd_releases.sh + +# Locale settings impact file ordering in ls or shell file expansion. The file order is used to +# generate files that are subsequently validated by the CI. If local environments use different +# locales to the CI we get unexpected failures that are tricky to debug without knowledge of +# locales so we'll explicitly warn here. +# In a AL2 container image (like builder base), LANG will be empty which is equivalent to posix +# In a AL2 (or other distro) full instance the LANG will be en-us.UTF-8 which produces different sorts +# On Mac, LANG will be en-us.UTF-8 but has a fix applied to sort to avoid the difference +.PHONY: ensure-locale +ensure-locale: + @if [ "Linux" = "$$(uname -s)" ]; then \ + LOCALE=$$(locale | grep LANG | cut -d= -f2 | tr -d '"' | tr '[:upper:]' '[:lower:]'); \ + if [[ "c.utf-8 posix" != *"$${LOCALE:-posix}"* ]]; then \ + echo WARNING: Environment locale set to $(LANG). On Linux systems this may create \ + non-deterministic behavior when running generation recipes. If the CI fails validation try \ + exporting LANG=C.UTF-8 to generate files instead.; \ + fi; \ + fi