From 3548eab07dca4a44f0262db93be013414ffa1cbc Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Tue, 26 Mar 2024 16:11:06 +0000 Subject: [PATCH] More controlled environment in the build Signed-off-by: Andrea Peruffo --- .dockerignore | 2 ++ .gitignore | 4 ++++ Dockerfile | 16 +++++--------- Makefile | 46 +++++++++++++++++++++++---------------- docker-compose-local.yaml | 16 ++++++++++---- scripts/install_protoc.sh | 20 +++++++++++++++++ 6 files changed, 70 insertions(+), 34 deletions(-) create mode 100644 .dockerignore create mode 100755 scripts/install_protoc.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..f732a0b2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +bin +include diff --git a/.gitignore b/.gitignore index 0d0c4601..01ceb11b 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,7 @@ log.html output.xml report.html __pycache__ + +# Protoc files +include/ +readme.txt diff --git a/Dockerfile b/Dockerfile index e12191d6..8f94195a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,25 +19,19 @@ RUN yum install -y nodejs npm java-11 # Copy the go source COPY ["Makefile", "main.go", ".openapi-generator-ignore", "openapitools.json", "./"] -# Download protoc compiler v24.3 -RUN wget -q https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip -O protoc.zip && \ - unzip -q protoc.zip && \ - bin/protoc --version && \ - rm protoc.zip - -# Download tools -RUN make deps - # Copy rest of the source -COPY bin/ bin/ +COPY .git/ .git/ COPY cmd/ cmd/ COPY api/ api/ COPY internal/ internal/ -COPY pkg/ pkg/ COPY scripts/ scripts/ +COPY pkg/ pkg/ COPY patches/ patches/ COPY templates/ templates/ +# Download tools +RUN make deps + # Build USER root RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 make clean model-registry diff --git a/Makefile b/Makefile index 24b1345d..20a65905 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) PROJECT_PATH := $(patsubst %/,%,$(dir $(MKFILE_PATH))) PROJECT_BIN := $(PROJECT_PATH)/bin +GO := $(PROJECT_BIN)/go1.19 # add tools bin directory PATH := $(PROJECT_BIN):$(PATH) @@ -56,7 +57,7 @@ api/grpc/ml_metadata/proto/metadata_store_service.proto: sed -i 's#syntax = "proto[23]";#&\noption go_package = "github.com/kubeflow/model-registry/internal/ml_metadata/proto";#' metadata_store_service.proto internal/ml_metadata/proto/%.pb.go: api/grpc/ml_metadata/proto/%.proto - protoc -I./api/grpc --go_out=./internal --go_opt=paths=source_relative \ + bin/protoc -I./api/grpc --go_out=./internal --go_opt=paths=source_relative \ --go-grpc_out=./internal --go-grpc_opt=paths=source_relative $< .PHONY: gen/grpc @@ -76,8 +77,8 @@ openapi/validate: bin/openapi-generator-cli # generate the openapi server implementation .PHONY: gen/openapi-server gen/openapi-server: bin/openapi-generator-cli openapi/validate - @if git diff --cached --name-only | grep -q "api/openapi/model-registry.yaml" || \ - git diff --name-only | grep -q "api/openapi/model-registry.yaml" || \ + @if git diff --exit-code --name-only | grep -q "api/openapi/model-registry.yaml" || \ + git diff --exit-code --name-only | grep -q "api/openapi/model-registry.yaml" || \ [ -n "${FORCE_SERVER_GENERATION}" ]; then \ ROOT_FOLDER="." ./scripts/gen_openapi_server.sh; \ else \ @@ -97,7 +98,7 @@ pkg/openapi/client.go: bin/openapi-generator-cli api/openapi/model-registry.yaml .PHONY: vet vet: - go vet ./... + ${GO} vet ./... .PHONY: clean clean: @@ -107,14 +108,21 @@ clean: clean/odh: rm -Rf ./model-registry +bin/go: + GOBIN=$(PROJECT_BIN) go install golang.org/dl/go1.19@latest + $(PROJECT_BIN)/go1.19 download + +bin/protoc: + ./scripts/install_protoc.sh + bin/go-enum: - GOBIN=$(PROJECT_BIN) go install github.com/searKing/golang/tools/go-enum@v1.2.97 + GOBIN=$(PROJECT_BIN) ${GO} install github.com/searKing/golang/tools/go-enum@v1.2.97 bin/protoc-gen-go: - GOBIN=$(PROJECT_BIN) go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31.0 + GOBIN=$(PROJECT_BIN) ${GO} install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31.0 bin/protoc-gen-go-grpc: - GOBIN=$(PROJECT_BIN) go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 + GOBIN=$(PROJECT_BIN) ${GO} install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 GOLANGCI_LINT ?= ${PROJECT_BIN}/golangci-lint bin/golangci-lint: @@ -122,7 +130,7 @@ bin/golangci-lint: GOVERTER ?= ${PROJECT_BIN}/goverter bin/goverter: - GOBIN=$(PROJECT_PATH)/bin go install github.com/jmattheis/goverter/cmd/goverter@v1.1.1 + GOBIN=$(PROJECT_PATH)/bin ${GO} install github.com/jmattheis/goverter/cmd/goverter@v1.1.1 OPENAPI_GENERATOR ?= ${PROJECT_BIN}/openapi-generator-cli NPM ?= "$(shell which npm)" @@ -147,23 +155,23 @@ clean/deps: rm -Rf bin/* .PHONY: deps -deps: bin/go-enum bin/protoc-gen-go bin/protoc-gen-go-grpc bin/golangci-lint bin/goverter bin/openapi-generator-cli +deps: bin/go bin/protoc bin/go-enum bin/protoc-gen-go bin/protoc-gen-go-grpc bin/golangci-lint bin/goverter bin/openapi-generator-cli .PHONY: vendor vendor: - go mod vendor + ${GO} mod vendor .PHONY: build build: gen vet lint - go build + ${GO} build -buildvcs=false .PHONY: build/odh build/odh: vet - go build + ${GO} build -buildvcs=false .PHONY: gen gen: deps gen/grpc gen/openapi gen/openapi-server gen/converter - go generate ./... + ${GO} generate ./... .PHONY: lint lint: @@ -172,20 +180,20 @@ lint: .PHONY: test test: gen - go test ./internal/... ./pkg/... + ${GO} test ./internal/... ./pkg/... .PHONY: test-nocache test-nocache: gen - go test ./internal/... ./pkg/... -count=1 + ${GO} test ./internal/... ./pkg/... -count=1 .PHONY: test-cover test-cover: gen - go test ./internal/... ./pkg/... -coverprofile=coverage.txt - go tool cover -html=coverage.txt -o coverage.html + ${GO} test ./internal/... ./pkg/... -coverprofile=coverage.txt + ${GO} tool cover -html=coverage.txt -o coverage.html .PHONY: run/proxy run/proxy: gen - go run main.go proxy --logtostderr=true + ${GO} run main.go proxy --logtostderr=true .PHONY: proxy proxy: build @@ -204,7 +212,7 @@ endif # build docker image .PHONY: image/build image/build: - ${DOCKER} build . -f ${DOCKERFILE} -t ${IMG}:$(IMG_VERSION) + ${DOCKER} build . -f ${DOCKERFILE} -t ${IMG}:$(IMG_VERSION) --network host # push docker image .PHONY: image/push diff --git a/docker-compose-local.yaml b/docker-compose-local.yaml index 52b7601f..7e41366a 100644 --- a/docker-compose-local.yaml +++ b/docker-compose-local.yaml @@ -1,10 +1,13 @@ -version: '3' +version: '3.4' services: mlmd-server: image: gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 container_name: mlmd-server ports: - - "9090:8080" + - target: 8080 + published: 9090 + protocol: tcp + mode: host environment: - METADATA_STORE_SERVER_CONFIG_FILE=/tmp/shared/conn_config.pb volumes: @@ -13,8 +16,13 @@ services: build: context: . dockerfile: Dockerfile - command: ["proxy", "--mlmd-hostname", "localhost", "--mlmd-port", "9090"] + network: host + command: ["proxy", "--hostname", "0.0.0.0", "--mlmd-hostname", "mlmd-server", "--mlmd-port", "8080"] container_name: model-registry - network_mode: host + ports: + - target: 8080 + published: 8080 + protocol: tcp + mode: host depends_on: - mlmd-server diff --git a/scripts/install_protoc.sh b/scripts/install_protoc.sh new file mode 100755 index 00000000..d8ce2a18 --- /dev/null +++ b/scripts/install_protoc.sh @@ -0,0 +1,20 @@ +#! /bin/bash +set -euxo pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +VERSION="24.3" +OS="linux" +if [[ "$OSTYPE" == "darwin"* ]]; then + # Mac OSX + OS="osx" +fi +# TODO: support for arm +ARCH="x86_64" + +mkdir -p ${SCRIPT_DIR}/../bin + +wget -q https://github.com/protocolbuffers/protobuf/releases/download/v${VERSION}/protoc-${VERSION}-${OS}-${ARCH}.zip -O ${SCRIPT_DIR}/../protoc.zip && \ + unzip -qo ${SCRIPT_DIR}/../protoc.zip -d ${SCRIPT_DIR}/.. && \ + bin/protoc --version && \ + rm ${SCRIPT_DIR}/../protoc.zip