Skip to content

Commit

Permalink
improves local check by turning into a prereq (#2536)
Browse files Browse the repository at this point in the history
  • Loading branch information
jaxesn authored Oct 11, 2023
1 parent e551242 commit c731efc
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 37 deletions.
39 changes: 20 additions & 19 deletions Common.mk
Original file line number Diff line number Diff line change
Expand Up @@ -437,23 +437,6 @@ BUILD_TARGETS?=github-rate-limit-pre validate-checksums attribution $(if $(IMAGE
RELEASE_TARGETS?=validate-checksums $(if $(IMAGE_NAMES),images,) $(if $(filter true,$(HAS_HELM_CHART)),helm/push,) $(if $(filter true,$(HAS_S3_ARTIFACTS)),upload-artifacts,)
####################################################

# Locale settings impact file ordering in ls or shell file expansion. The file order is used to
# generate files that are subsequently validated by the CI. If local environments use different
# locales to the CI we get unexpected failures that are tricky to debug without knowledge of
# locales so we'll explicitly warn here.
# In a AL2 container image (like builder base), LANG will be empty which is equilvant to posix
# In a AL2 (or other distro) full instance the LANG will be en-us.UTF-8 which produces different sorts
# On Mac, LANG will be en-us.UTF-8 but has a fix applied to sort to avoid the difference
ifeq ($(shell uname -s),Linux)
LOCALE:=$(call TO_LOWER,$(shell locale | grep LANG | cut -d= -f2 | tr -d '"'))
LOCALE:=$(if $(LOCALE),$(LOCALE),posix)
ifeq ($(filter c.utf-8 posix,$(LOCALE)),)
$(warning WARNING: Environment locale set to $(LANG). On Linux systems this may create \
non-deterministic behavior when running generation recipes. If the CI fails validation try \
`LANG=C.UTF-8 make <recipe>` to generate files instead.)
endif
endif

define BUILDCTL
$(BUILD_LIB)/buildkit.sh \
build \
Expand Down Expand Up @@ -621,7 +604,7 @@ $(OUTPUT_DIR)/%TTRIBUTION.txt:
$(OUTPUT_DIR)/%ttribution/go-license.csv: BINARY_TARGET=$(if $(filter .,$(*D)),,$(*D))
$(OUTPUT_DIR)/%ttribution/go-license.csv: GO_MOD_PATH=$(if $(BINARY_TARGET),$(GO_MOD_TARGET_FOR_BINARY_$(call TO_UPPER,$(BINARY_TARGET))),$(word 1,$(UNIQ_GO_MOD_PATHS)))
$(OUTPUT_DIR)/%ttribution/go-license.csv: LICENSE_PACKAGE_FILTER=$(GO_MOD_$(subst /,_,$(GO_MOD_PATH))_LICENSE_PACKAGE_FILTER)
$(OUTPUT_DIR)/%ttribution/go-license.csv: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH))
$(OUTPUT_DIR)/%ttribution/go-license.csv: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH))
@echo -e $(call TARGET_START_LOG)
$(BASE_DIRECTORY)/build/lib/gather_licenses.sh $(REPO) $(MAKE_ROOT)/$(OUTPUT_DIR)/$(BINARY_TARGET) "$(LICENSE_PACKAGE_FILTER)" $(GO_MOD_PATH) $(GOLANG_VERSION) $(LICENSE_THRESHOLD)
@echo -e $(call TARGET_END_LOG)
Expand Down Expand Up @@ -913,7 +896,7 @@ stop-docker-builder: # Clean up builder base docker container
docker rm -f -v eks-a-builder

.PHONY: generate
generate: # Update UPSTREAM_PROJECTS.yaml
generate: | ensure-locale # Update UPSTREAM_PROJECTS.yaml
$(BUILD_LIB)/generate_projects_list.sh $(BASE_DIRECTORY)

.PHONY: update-go-mods
Expand Down Expand Up @@ -988,6 +971,24 @@ github-rate-limit-%:
GH_PAGER='' gh api rate_limit; \
fi

# Locale settings impact file ordering in ls or shell file expansion. The file order is used to
# generate files that are subsequently validated by the CI. If local environments use different
# locales to the CI we get unexpected failures that are tricky to debug without knowledge of
# locales so we'll explicitly warn here.
# In a AL2 container image (like builder base), LANG will be empty which is equivalent to posix
# In a AL2 (or other distro) full instance the LANG will be en-us.UTF-8 which produces different sorts
# On Mac, LANG will be en-us.UTF-8 but has a fix applied to sort to avoid the difference
.PHONY: ensure-locale
ensure-locale:
@if [ "Linux" = "$$(uname -s)" ]; then \
LOCALE=$$(locale | grep LANG | cut -d= -f2 | tr -d '"' | tr '[:upper:]' '[:lower:]'); \
if [[ "c.utf-8 posix" != *"$${LOCALE:-posix}"* ]]; then \
echo WARNING: Environment locale set to $(LANG). On Linux systems this may create \
non-deterministic behavior when running generation recipes. If the CI fails validation try \
exporting LANG=C.UTF-8 to generate files instead.; \
fi; \
fi

## --------------------------------------
## Docker Helpers
## --------------------------------------
Expand Down
38 changes: 20 additions & 18 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,22 +13,6 @@ ALL_PROJECTS=$(shell $(BUILD_LIB)/all_projects.sh $(BASE_DIRECTORY))
# $1 - project name using _ as separator, ex: rancher_local-path-provisoner
PROJECT_PATH_MAP=projects/$(patsubst $(firstword $(subst _, ,$(1)))_%,$(firstword $(subst _, ,$(1)))/%,$(1))

# Locale settings impact file ordering in ls or shell file expansion. The file order is used to
# generate files that are subsequently validated by the CI. If local environments use different
# locales to the CI we get unexpected failures that are tricky to debug without knowledge of
# locales so we'll explicitly warn here.
# In a AL2 container image (like builder base), LANG will be empty which is equivalent to posix
# In a AL2 (or other distro) full instance the LANG will be en-us.UTF-8 which produces different sorts
# On Mac, LANG will be en-us.UTF-8 but has a fix applied to sort to avoid the difference
ifeq ($(shell uname -s),Linux)
LOCALE:=$(call TO_LOWER,$(shell locale | grep LANG | cut -d= -f2 | tr -d '"'))
LOCALE:=$(if $(LOCALE),$(LOCALE),posix)
ifeq ($(filter c.utf-8 posix,$(LOCALE)),)
$(warning WARNING: Environment locale set to $(LANG). On Linux systems this may create \
non-deterministic behavior when running generation recipes. If the CI fails validation try \
`LANG=C.UTF-8 make <recipe>` to generate files instead.)
endif
endif

.PHONY: clean-project-%
clean-project-%:
Expand Down Expand Up @@ -86,11 +70,11 @@ stop-buildkit-and-registry:
docker rm -v --force registry

.PHONY: generate-project-list
generate-project-list:
generate-project-list: | ensure-locale
build/lib/generate_projects_list.sh $(BASE_DIRECTORY)

.PHONY: generate-staging-buildspec
generate-staging-buildspec:
generate-staging-buildspec: | ensure-locale
build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/release/staging-build.yml"
build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/release/checksums-build.yml" true EXCLUDE_FROM_CHECKSUMS_BUILDSPEC CHECKSUMS_BUILDSPECS false buildspecs/checksums-pr-buildspec.yml
build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "aws_bottlerocket-bootstrap" "$(BASE_DIRECTORY)/projects/aws/bottlerocket-bootstrap/buildspecs/batch-build.yml" true
Expand Down Expand Up @@ -122,3 +106,21 @@ check-project-path-exists:
.PHONY: validate-eksd-releases
validate-eksd-releases:
build/lib/validate_eksd_releases.sh

# Locale settings impact file ordering in ls or shell file expansion. The file order is used to
# generate files that are subsequently validated by the CI. If local environments use different
# locales to the CI we get unexpected failures that are tricky to debug without knowledge of
# locales so we'll explicitly warn here.
# In a AL2 container image (like builder base), LANG will be empty which is equivalent to posix
# In a AL2 (or other distro) full instance the LANG will be en-us.UTF-8 which produces different sorts
# On Mac, LANG will be en-us.UTF-8 but has a fix applied to sort to avoid the difference
.PHONY: ensure-locale
ensure-locale:
@if [ "Linux" = "$$(uname -s)" ]; then \
LOCALE=$$(locale | grep LANG | cut -d= -f2 | tr -d '"' | tr '[:upper:]' '[:lower:]'); \
if [[ "c.utf-8 posix" != *"$${LOCALE:-posix}"* ]]; then \
echo WARNING: Environment locale set to $(LANG). On Linux systems this may create \
non-deterministic behavior when running generation recipes. If the CI fails validation try \
exporting LANG=C.UTF-8 to generate files instead.; \
fi; \
fi

0 comments on commit c731efc

Please sign in to comment.