Skip to content

Commit

Permalink
refactors logging to use annonation style
Browse files Browse the repository at this point in the history
  • Loading branch information
jaxesn committed Oct 12, 2023
1 parent 51dff6b commit 42bff5a
Show file tree
Hide file tree
Showing 7 changed files with 112 additions and 100 deletions.
86 changes: 37 additions & 49 deletions Common.mk
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,17 @@ DATE_CMD=TZ=utc $(shell source $(BUILD_LIB)/common.sh && build::find::gnu_varian
DATE_NANO=$(shell if [ "$$(uname -s)" = "Linux" ] || command -v gdate &> /dev/null; then echo %3N; fi)
TARGET_START_LOG?=$(eval _START_TIME:=$(shell $(DATE_CMD) +%s.$(DATE_NANO)))\\n------------------- $(shell $(DATE_CMD) +"%Y-%m-%dT%H:%M:%S.$(DATE_NANO)%z") Starting target=$@ -------------------
TARGET_END_LOG?="------------------- `$(DATE_CMD) +'%Y-%m-%dT%H:%M:%S.$(DATE_NANO)%z'` Finished target=$@ duration=`echo $$($(DATE_CMD) +%s.$(DATE_NANO)) - $(_START_TIME) | bc` seconds -------------------\\n"

ENABLE_LOGGING=$(eval $(call ENABLE_LOGGING_BODY,$@))
# $1 - target name
# target is exported so that the script can tell the difference between
# a recipe and a $(shell) call, which we do not want to log around
# this style of enable logging only works for single line recipes
define ENABLE_LOGGING_BODY
$(eval _TARGET:=$1)
$(_TARGET): export TARGET=$(_TARGET)
$(_TARGET): SHELL=$(BUILD_LIB)/make_logging_shell.sh
endef
####################################################

#################### TARGETS FOR OVERRIDING ########
Expand Down Expand Up @@ -473,11 +484,11 @@ endef
# This will occansionally stall out in codebuild for an unknown reason
# retry after a configurable timeout
define CGO_DOCKER
$(BUILD_LIB)/run_target_docker.sh $(COMPONENT) $(CGO_TARGET) $(IMAGE_REPO) "$(RELEASE_BRANCH)" "$(ARTIFACTS_BUCKET)" "$(BASE_DIRECTORY)" "$(GO_MOD_CACHE)" true "$(IMAGE_PLATFORMS)"
@$(BUILD_LIB)/run_target_docker.sh $(COMPONENT) $(CGO_TARGET) $(IMAGE_REPO) "$(RELEASE_BRANCH)" "$(ARTIFACTS_BUCKET)" "$(BASE_DIRECTORY)" "$(GO_MOD_CACHE)" true "$(IMAGE_PLATFORMS)"
endef

define SIMPLE_CREATE_BINARIES_SHELL
$(BASE_DIRECTORY)/build/lib/simple_create_binaries.sh $(MAKE_ROOT) $(MAKE_ROOT)/$(OUTPUT_PATH) $(REPO) $(GOLANG_VERSION) $(PLATFORM) "$(SOURCE_PATTERN)" \
@$(BASE_DIRECTORY)/build/lib/simple_create_binaries.sh $(MAKE_ROOT) $(MAKE_ROOT)/$(OUTPUT_PATH) $(REPO) $(GOLANG_VERSION) $(PLATFORM) "$(SOURCE_PATTERN)" \
"$(GOBUILD_COMMAND)" "$(EXTRA_GOBUILD_FLAGS)" "$(GO_LDFLAGS)" $(CGO_ENABLED) "$(CGO_LDFLAGS)" "$(GO_MOD_PATH)" "$(BINARY_TARGET_FILES_BUILD_TOGETHER)"
endef

Expand Down Expand Up @@ -553,7 +564,7 @@ $(HELM_GIT_CHECKOUT_TARGET): | $(HELM_SOURCE_REPOSITORY)
(cd $(HELM_SOURCE_REPOSITORY) && $(BASE_DIRECTORY)/build/lib/wait_for_tag.sh $(HELM_GIT_TAG))
git -C $(HELM_SOURCE_REPOSITORY) checkout -f $(HELM_GIT_TAG)
touch $@
@echo -e $(call TARGET_END_LOG)
@echo -e $(call TARGET_END_LOG)
endif

$(HELM_GIT_PATCH_TARGET): $(HELM_GIT_CHECKOUT_TARGET)
Expand All @@ -572,10 +583,8 @@ $(OUTPUT_BIN_DIR)/%: BINARY_TARGET=$(@F:%.exe=%)
$(OUTPUT_BIN_DIR)/%: SOURCE_PATTERN=$(if $(filter $(BINARY_TARGET),$(BINARY_TARGET_FILES_BUILD_TOGETHER)),$(SOURCE_PATTERNS_BUILD_TOGETHER),$(word $(call pos,$(BINARY_TARGET),$(BINARY_TARGET_FILES)),$(SOURCE_PATTERNS)))
$(OUTPUT_BIN_DIR)/%: OUTPUT_PATH=$(if $(and $(if $(filter false,$(call IS_ONE_WORD,$(BINARY_TARGET_FILES_BUILD_TOGETHER))),$(filter $(BINARY_TARGET),$(BINARY_TARGET_FILES_BUILD_TOGETHER)))),$(@D)/,$@)
$(OUTPUT_BIN_DIR)/%: GO_MOD_PATH=$($(call GO_MOD_TARGET_FOR_BINARY_VAR_NAME,$(BINARY_TARGET)))
$(OUTPUT_BIN_DIR)/%: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH))
@echo -e $(call TARGET_START_LOG)
$(OUTPUT_BIN_DIR)/%: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH)) | $$(ENABLE_LOGGING)
$(if $(NEEDS_CGO_BUILDER),$(call CGO_CREATE_BINARIES_SHELL,$@,$(*D)),$(call SIMPLE_CREATE_BINARIES_SHELL))
@echo -e $(call TARGET_END_LOG)
endif

.PHONY: binaries
Expand Down Expand Up @@ -619,10 +628,8 @@ $(OUTPUT_DIR)/%TTRIBUTION.txt:
$(OUTPUT_DIR)/%ttribution/go-license.csv: BINARY_TARGET=$(if $(filter .,$(*D)),,$(*D))
$(OUTPUT_DIR)/%ttribution/go-license.csv: GO_MOD_PATH=$(if $(BINARY_TARGET),$(GO_MOD_TARGET_FOR_BINARY_$(call TO_UPPER,$(BINARY_TARGET))),$(word 1,$(UNIQ_GO_MOD_PATHS)))
$(OUTPUT_DIR)/%ttribution/go-license.csv: LICENSE_PACKAGE_FILTER=$(GO_MOD_$(subst /,_,$(GO_MOD_PATH))_LICENSE_PACKAGE_FILTER)
$(OUTPUT_DIR)/%ttribution/go-license.csv: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH)) | ensure-jq
@echo -e $(call TARGET_START_LOG)
$(BASE_DIRECTORY)/build/lib/gather_licenses.sh $(REPO) $(MAKE_ROOT)/$(OUTPUT_DIR)/$(BINARY_TARGET) "$(LICENSE_PACKAGE_FILTER)" $(GO_MOD_PATH) $(GOLANG_VERSION) $(LICENSE_THRESHOLD)
@echo -e $(call TARGET_END_LOG)
$(OUTPUT_DIR)/%ttribution/go-license.csv: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH)) | ensure-jq $$(ENABLE_LOGGING)
@$(BASE_DIRECTORY)/build/lib/gather_licenses.sh $(REPO) $(MAKE_ROOT)/$(OUTPUT_DIR)/$(BINARY_TARGET) "$(LICENSE_PACKAGE_FILTER)" $(GO_MOD_PATH) $(GOLANG_VERSION) $(LICENSE_THRESHOLD)

.PHONY: gather-licenses
gather-licenses: $(GATHER_LICENSES_TARGETS)
Expand All @@ -631,20 +638,15 @@ gather-licenses: $(GATHER_LICENSES_TARGETS)
# if there is only one go mod path so only one attribution is created, the file will be named ATTRIBUTION.txt and licenses will be stored in _output, `%` will equal `A`
# if multiple attributions are being generated, the file will be <binary>_ATTRIBUTION.txt and licenses will be stored in _output/<binary>, `%` will equal `<BINARY>_A`
%TTRIBUTION.txt: LICENSE_OUTPUT_PATH=$(OUTPUT_DIR)$(if $(filter A,$(*F)),,/$(call TO_LOWER,$(*F:%_A=%)))
%TTRIBUTION.txt: $$(LICENSE_OUTPUT_PATH)/attribution/go-license.csv
@echo -e $(call TARGET_START_LOG)
@rm -f $(@F)
$(BASE_DIRECTORY)/build/lib/create_attribution.sh $(MAKE_ROOT) $(GOLANG_VERSION) $(MAKE_ROOT)/$(LICENSE_OUTPUT_PATH) $(@F) $(RELEASE_BRANCH)
@echo -e $(call TARGET_END_LOG)
%TTRIBUTION.txt: $$(LICENSE_OUTPUT_PATH)/attribution/go-license.csv | $$(ENABLE_LOGGING)
@$(BASE_DIRECTORY)/build/lib/create_attribution.sh $(MAKE_ROOT) $(GOLANG_VERSION) $(MAKE_ROOT)/$(LICENSE_OUTPUT_PATH) $(@F) $(RELEASE_BRANCH)

.PHONY: attribution
attribution: $(and $(filter true,$(HAS_LICENSES)),$(ATTRIBUTION_TARGETS))

.PHONY: attribution-pr
attribution-pr: attribution
@echo -e $(call TARGET_START_LOG)
$(BASE_DIRECTORY)/build/update-attribution-files/create_pr.sh
@echo -e $(call TARGET_END_LOG)
attribution-pr: attribution | $$(ENABLE_LOGGING)
@$(BASE_DIRECTORY)/build/update-attribution-files/create_pr.sh

.PHONY: all-attributions
all-attributions:
Expand All @@ -653,18 +655,14 @@ all-attributions:
#### Tarball Targets

.PHONY: tarballs
tarballs: $(LICENSES_TARGETS_FOR_PREREQ)
tarballs: $(LICENSES_TARGETS_FOR_PREREQ) | $$(ENABLE_LOGGING)
ifeq ($(SIMPLE_CREATE_TARBALLS),true)
@echo -e $(call TARGET_START_LOG)
$(BASE_DIRECTORY)/build/lib/simple_create_tarballs.sh $(TAR_FILE_PREFIX) $(MAKE_ROOT)/$(OUTPUT_DIR) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR) $(GIT_TAG) "$(BINARY_PLATFORMS)" $(ARTIFACTS_PATH) $(GIT_HASH)
@echo -e $(call TARGET_END_LOG)
@$(BASE_DIRECTORY)/build/lib/simple_create_tarballs.sh $(TAR_FILE_PREFIX) $(MAKE_ROOT)/$(OUTPUT_DIR) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR) $(GIT_TAG) "$(BINARY_PLATFORMS)" $(ARTIFACTS_PATH) $(GIT_HASH)
endif

.PHONY: upload-artifacts
upload-artifacts: s3-artifacts upload-output-to-prow-artifacts-s3-artifacts
@echo -e $(call TARGET_START_LOG)
$(BASE_DIRECTORY)/build/lib/upload_artifacts.sh $(ARTIFACTS_PATH) $(ARTIFACTS_BUCKET) $(ARTIFACTS_UPLOAD_PATH) $(BUILD_IDENTIFIER) $(GIT_HASH) $(LATEST) $(UPLOAD_DRY_RUN) $(UPLOAD_DO_NOT_DELETE)
@echo -e $(call TARGET_END_LOG)
upload-artifacts: s3-artifacts upload-output-to-prow-artifacts-s3-artifacts | $$(ENABLE_LOGGING)
@$(BASE_DIRECTORY)/build/lib/upload_artifacts.sh $(ARTIFACTS_PATH) $(ARTIFACTS_BUCKET) $(ARTIFACTS_UPLOAD_PATH) $(BUILD_IDENTIFIER) $(GIT_HASH) $(LATEST) $(UPLOAD_DRY_RUN) $(UPLOAD_DO_NOT_DELETE)

.PHONY: s3-artifacts
s3-artifacts: tarballs
Expand All @@ -682,19 +680,15 @@ upload-output-to-prow-artifacts-%:
### Checksum Targets

.PHONY: checksums
checksums: $(BINARY_TARGETS)
checksums: $(BINARY_TARGETS) | $$(ENABLE_LOGGING)
ifneq ($(strip $(BINARY_TARGETS)),)
@echo -e $(call TARGET_START_LOG)
$(BASE_DIRECTORY)/build/lib/update_checksums.sh $(MAKE_ROOT) $(PROJECT_ROOT) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR)
@echo -e $(call TARGET_END_LOG)
@$(BASE_DIRECTORY)/build/lib/update_checksums.sh $(MAKE_ROOT) $(PROJECT_ROOT) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR)
endif

.PHONY: validate-checksums
validate-checksums: $(BINARY_TARGETS) upload-output-to-prow-artifacts-validate-checksums
validate-checksums: $(BINARY_TARGETS) upload-output-to-prow-artifacts-validate-checksums | $$(ENABLE_LOGGING)
ifneq ($(and $(strip $(BINARY_TARGETS)), $(filter false, $(SKIP_CHECKSUM_VALIDATION))),)
@echo -e $(call TARGET_START_LOG)
$(BASE_DIRECTORY)/build/lib/validate_checksums.sh $(MAKE_ROOT) $(PROJECT_ROOT) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR) $(FAKE_ARM_BINARIES_FOR_VALIDATION) $(FAKE_AMD_BINARIES_FOR_VALIDATION)
@echo -e $(call TARGET_END_LOG)
endif

.PHONY: attribution-checksums
Expand Down Expand Up @@ -742,10 +736,8 @@ clean-job-caches: $(and $(findstring presubmit,$(JOB_TYPE)),$(filter true,$(PRUN
%/images/amd64 %/images/arm64: IMAGE_OUTPUT_TYPE?=oci
%/images/amd64 %/images/arm64: IMAGE_OUTPUT?=dest=$(IMAGE_OUTPUT_DIR)/$(IMAGE_OUTPUT_NAME).tar

%/images/push: $(BINARY_TARGETS) $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET) | ensure-buildctl ensure-buildkitd-host
@echo -e $(call TARGET_START_LOG)
$(BUILDCTL)
@echo -e $(call TARGET_END_LOG)
%/images/push: $(BINARY_TARGETS) $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET) | ensure-buildctl ensure-buildkitd-host $$(ENABLE_LOGGING)
@$(BUILDCTL)

%/images/amd64: $(BINARY_TARGETS) $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET) | ensure-buildctl ensure-buildkitd-host
@echo -e $(call TARGET_START_LOG)
Expand Down Expand Up @@ -796,10 +788,8 @@ binary-builder/cgo/%: USE_DOCKER_FOR_CGO_BUILD=$(shell command -v docker &> /dev

## Helm Targets
.PHONY: helm/pull
helm/pull:
@echo -e $(call TARGET_START_LOG)
$(BUILD_LIB)/helm_pull.sh $(HELM_PULL_LOCATION) $(HELM_REPO_URL) $(HELM_PULL_NAME) $(REPO) $(HELM_DIRECTORY) $(CHART_VERSION) $(COPY_CRDS)
@echo -e $(call TARGET_END_LOG)
helm/pull: | $$(ENABLE_LOGGING)
@$(BUILD_LIB)/helm_pull.sh $(HELM_PULL_LOCATION) $(HELM_REPO_URL) $(HELM_PULL_NAME) $(REPO) $(HELM_DIRECTORY) $(CHART_VERSION) $(COPY_CRDS)

# Build helm chart
.PHONY: helm/build
Expand All @@ -815,19 +805,15 @@ helm/build: $(if $(wildcard $(PROJECT_ROOT)/helm/patches),$(HELM_GIT_PATCH_TARGE

# Build helm chart and push to registry defined in IMAGE_REPO.
.PHONY: helm/push
helm/push: helm/build | ensure-helm
@echo -e $(call TARGET_START_LOG)
$(BUILD_LIB)/helm_push.sh $(IMAGE_REPO) $(HELM_DESTINATION_REPOSITORY) $(HELM_TAG) $(GIT_TAG) $(OUTPUT_DIR) $(LATEST)
@echo -e $(call TARGET_END_LOG)
helm/push: helm/build | ensure-helm $$(ENABLE_LOGGING)
@$(BUILD_LIB)/helm_push.sh $(IMAGE_REPO) $(HELM_DESTINATION_REPOSITORY) $(HELM_TAG) $(GIT_TAG) $(OUTPUT_DIR) $(LATEST)

## Fetch Binary Targets
.PHONY: handle-dependencies
handle-dependencies: $(call PROJECT_DEPENDENCIES_TARGETS)

$(BINARY_DEPS_DIR)/linux-%:
@echo -e $(call TARGET_START_LOG)
$(BINARY_DEPS_DIR)/linux-%: | $$(ENABLE_LOGGING)
$(BUILD_LIB)/fetch_binaries.sh $(BINARY_DEPS_DIR) $* $(ARTIFACTS_BUCKET) $(LATEST) $(RELEASE_BRANCH)
@echo -e $(call TARGET_END_LOG)

## Build Targets
.PHONY: build
Expand Down Expand Up @@ -1062,3 +1048,5 @@ run-$(1)-in-docker: run-target-in-docker
endef

$(foreach target,$(IN_DOCKER_TARGETS),$(eval $(call RUN_IN_DOCKER_TARGET,$(target))))

%: SHELL=bash
10 changes: 9 additions & 1 deletion build/lib/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# Short-circuit if script has already been sourced
[[ $(type -t build::common::loaded) == function ]] && return 0

BUILD_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/" && pwd -P)"
source "${BUILD_ROOT}/eksd_releases.sh"

Expand Down Expand Up @@ -149,7 +152,7 @@ function build::gather_licenses() {
# data about each dependency to generate the amazon approved attribution.txt files
# go-deps is needed for module versions
# go-licenses are all the dependencies found from the module(s) that were passed in via patterns
build::common::echo_and_run go list -deps=true -json ./... | jq -s '' > "${outputdir}/attribution/go-deps.json"
build::common::echo_and_run go list -deps=true -json ./... | jq -s '.' > "${outputdir}/attribution/go-deps.json"

# go-licenses can be a bit noisy with its output and lot of it can be confusing
# the following messages are safe to ignore since we do not need the license url for our process
Expand Down Expand Up @@ -454,3 +457,8 @@ function build::common::copy_if_source_destination_different() {

cp -rf $source $destination
}

# Marker function to indicate script has been fully sourced
function build::common::loaded() {
return 0
}
2 changes: 2 additions & 0 deletions build/lib/create_attribution.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ OUTPUT_DIR="$3"
OUTPUT_FILENAME="$4"
RELEASE_BRANCH="${5:-}"

rm -f $OUTPUT_FILENAME

if [[ -n "$RELEASE_BRANCH" ]] && [ -d "$PROJECT_ROOT/$RELEASE_BRANCH" ]; then
PROJECT_ROOT=$PROJECT_ROOT/$RELEASE_BRANCH
fi
Expand Down
39 changes: 39 additions & 0 deletions build/lib/make_logging_shell.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# remove -c passed by make
shift

if [ -z "${TARGET:-}" ]; then
eval "$@"
exit $?
fi

set -o errexit
set -o nounset
set -o pipefail

SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
source "${SCRIPT_ROOT}/common.sh"

DATE=$(build::find::gnu_variant_on_mac date)
DATE_NANO=$(if [ "$(uname -s)" = "Linux" ] || [ "$DATE" = "gdate" ]; then echo %3N; fi)

START_TIME=$($DATE +%s.$DATE_NANO)

echo -e "\n------------------- $($DATE +"%Y-%m-%dT%H:%M:%S.$DATE_NANO%z") Starting target=$TARGET -------------------"
echo "($(pwd)) \$ $@"
eval "$@"
echo -e "------------------- $($DATE +"%Y-%m-%dT%H:%M:%S.$DATE_NANO%z") Finished target=$TARGET duration=$(echo $($DATE +%s.$DATE_NANO) - $START_TIME | bc) seconds -------------------\n"
6 changes: 2 additions & 4 deletions projects/containerd/containerd/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,8 @@ SIMPLE_CREATE_TARBALLS=false
include $(BASE_DIRECTORY)/Common.mk

.PHONY: tarballs
tarballs: $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET)
@echo -e $(call TARGET_START_LOG)
build/create_tarballs.sh $(TAR_FILE_PREFIX) $(MAKE_ROOT)/$(OUTPUT_DIR) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR) $(GIT_TAG) "$(BINARY_PLATFORMS)" $(ARTIFACTS_PATH) $(GIT_HASH) $(MAKE_ROOT)/$(BINARY_DEPS_DIR)
@echo -e $(call TARGET_END_LOG)
tarballs: $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET) | $$(ENABLE_LOGGING)
@build/create_tarballs.sh $(TAR_FILE_PREFIX) $(MAKE_ROOT)/$(OUTPUT_DIR) $(MAKE_ROOT)/$(OUTPUT_BIN_DIR) $(GIT_TAG) "$(BINARY_PLATFORMS)" $(ARTIFACTS_PATH) $(GIT_HASH) $(MAKE_ROOT)/$(BINARY_DEPS_DIR)


########### DO NOT EDIT #############################
Expand Down
Loading

0 comments on commit 42bff5a

Please sign in to comment.