feat(cli): cli as k8s deployment for debugging (#6090)

* move oauth config to tls pkg

* use tls module in cli

* adjust docs for changes to cli

* add dockerfile for cli

* tidy up dockerfile for operator

* add makefile targets for deploy / undeploy cli

* set scheduler port from existing envar

* add kafka config component

* use component kafka config

* add dependabot for component kafka

* add gh test for components (kafka and tls)

* build licenses for component kafka

* add docker push seldon cli

* refactor utility func to kafka component

* refactor consumer config into a separate func

* use kafka config map in cli

* wire up kafka config to seldon-cli path

* deal with kafka values as strings

* build cli docker image as part of ci

* add force flag for control plane load / unload

* add default for force in msg

* add k6 image building to CI

* add k8s in warning

* update docs

* review comments

.github/dependabot.yml

@@ -67,6 +67,17 @@ updates:
       - "dependencies"
     reviewers:
       - sakoush
+
+  - package-ecosystem: "gomod"
+    directory: "/components/kafka"
+    target-branch: "v2"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "v2"
+      - "dependencies"
+    reviewers:
+      - sakoush
 
   - package-ecosystem: "gradle"
     directory: "/scheduler/data-flow"

.github/workflows/images.yml

@@ -56,10 +56,18 @@ jobs:
         working-directory: ./operator
         run: CUSTOM_IMAGE_TAG=${{ steps.docker-tag.outputs.value }} make docker-build docker-push
 
+      - name: Push Docker Image for Seldon Cli
+        working-directory: ./operator
+        run: CUSTOM_IMAGE_TAG=${{ steps.docker-tag.outputs.value }} make docker-build-cli docker-push-cli
+
       - name: Push Docker Images for Scheduler
         working-directory: ./scheduler
         run: CUSTOM_IMAGE_TAG=${{ steps.docker-tag.outputs.value }} make docker-build-all docker-push-all
 
       - name: Push Docker Images for Hodometer
         working-directory: ./hodometer
         run: BUILD_VERSION=${{ steps.docker-tag.outputs.value }} IMAGE_TAG=${{ steps.docker-tag.outputs.value }} make build-hodometer-docker push-hodometer-docker
+
+      - name: Push Docker Image for k6 Load Testing
+        working-directory: ./tests/k6
+        run: CUSTOM_IMAGE_TAG=${{ steps.docker-tag.outputs.value }} make docker-build docker-push

.github/workflows/tests.yml

@@ -18,3 +18,7 @@ jobs:
         run: make -C scheduler test
       - name: test-hodometer
         run: make -C hodometer test
+      - name: test-components-tls
+        run: make -C components/tls test
+      - name: test-components-kafka
+        run: make -C components/kafka test

Makefile

@@ -111,9 +111,10 @@ update-3rd-party-licenses:
 	make -C operator licenses
 	make -C hodometer licenses
 	make -C components/tls licenses
+	make -C components/kafka licenses
 	make -C tests/integration licenses
 	make -C scheduler/data-flow licenses
-	{ (cat scheduler/licenses/license_info.csv operator/licenses/license_info.csv hodometer/licenses/license_info.csv components/tls/licenses/license_info.csv tests/integration/licenses/license_info.csv | cut -d, -f3) ; (cat scheduler/data-flow/licenses/dependency-license.json | jq .dependencies[].licenses[0].name) } | sed 's/\"//g' | sort | uniq -c > licenses/3rd-party-summary.txt
+	{ (cat scheduler/licenses/license_info.csv operator/licenses/license_info.csv hodometer/licenses/license_info.csv components/tls/licenses/license_info.csv components/kafka/licenses/license_info.csv tests/integration/licenses/license_info.csv | cut -d, -f3) ; (cat scheduler/data-flow/licenses/dependency-license.json | jq .dependencies[].licenses[0].name) } | sed 's/\"//g' | sort | uniq -c > licenses/3rd-party-summary.txt
 
 docs-gitbook-update-code-references:
 	python hack/update_references.py

components/kafka/.errcheck_excludes.txt

@@ -0,0 +1,2 @@
+fmt.Fprintln
+fmt.Fprint

components/kafka/.gitignore

@@ -0,0 +1 @@
+cover.out

components/kafka/.golangci.yml

@@ -0,0 +1,53 @@
+# options for analysis running
+run:
+  # timeout for analysis, e.g. 30s, 5m, default is 1m
+  deadline: 5m
+
+  # exit code when at least one issue was found, default is 1
+  issues-exit-code: 1
+
+  # which dirs to skip: they won't be analyzed;
+  # can use regexp here: generated.*, regexp is applied on full path;
+  # default value is empty list, but next dirs are always skipped independently
+  # from this option's value:
+  #     vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  skip-dirs: vendor
+
+# output configuration options
+output:
+  # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number"
+  format: colored-line-number
+
+  # print lines of code with issue, default is true
+  print-issued-lines: true
+
+  # print linter name in the end of issue text, default is true
+  print-linter-name: true
+
+linters:
+  disable-all: true
+  enable:
+    # Sorted alphabetically.
+    - errcheck
+    - exportloopref
+    - gci
+    - goimports # Also includes gofmt style formatting
+    - gosimple
+    - govet
+    - misspell
+    - staticcheck
+    - structcheck
+    - typecheck
+    - varcheck
+
+linters-settings:
+  errcheck:
+    exclude: ./.errcheck_excludes.txt
+  gci:
+    sections:
+      - standard
+      - default
+      - prefix(github.com/seldonio/seldon-core)
+      - prefix(github.com/seldonio/seldon-core/components/tls)
+  goconst:
+    min-occurrences: 5

components/kafka/Makefile

@@ -0,0 +1,38 @@
+GO_LDFLAGS := -w $(patsubst %,-X %, $(GO_BUILD_VARS))
+
+.PHONY: test
+test:
+	go test ./pkg/... -coverprofile cover.out
+
+.GOLANGCILINT_VERSION := v1.57.2
+.GOLANGCILINT_PATH := $(shell go env GOPATH)/bin/golangci-lint/$(.GOLANGCILINT_VERSION)
+
+${.GOLANGCILINT_PATH}/golangci-lint: 
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \
+			| sh -s -- -b ${.GOLANGCILINT_PATH} ${.GOLANGCILINT_VERSION}
+
+.PHONY: lint
+lint: ${.GOLANGCILINT_PATH}/golangci-lint
+	gofmt -w pkg
+	${.GOLANGCILINT_PATH}/golangci-lint run --fix
+
+.PHONY: build
+build: test
+	go build -trimpath -ldflags="-w" ./pkg/...
+
+.PHONY: licenses/dep.txt
+licenses/dep.txt:
+	go list -m all | cut -d ' ' -f 1 > licenses/dep.txt
+
+.PHONY: licenses
+licenses: licenses/dep.txt
+	# NOTE: You need to create a file in ~/.github_api_token with a GitHub token.
+	get-github-repo \
+		-o licenses/repo.txt \
+		--manual-dep-repo-mapping ../../licenses/dep_repo.manual.csv \
+		licenses/dep.txt
+	get-github-license-info -o licenses/license_info.csv licenses/repo.txt --branch-refs ../../licenses/branch_refs.manual.csv
+	python -m 'patch_additional_license_info' \
+		licenses/license_info.csv \
+		../../licenses/additional_license_info.csv
+	concatenate-license -o licenses/license.txt licenses/license_info.csv

components/kafka/go.mod

@@ -0,0 +1,17 @@
+module github.com/seldonio/seldon-core/components/kafka/v2
+
+go 1.22.7
+
+toolchain go1.22.9
+
+require (
+	github.com/confluentinc/confluent-kafka-go/v2 v2.6.1
+	github.com/onsi/gomega v1.35.1
+)
+
+require (
+	github.com/google/go-cmp v0.6.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

components/kafka/go.sum

@@ -0,0 +1,13 @@
+github.com/confluentinc/confluent-kafka-go/v2 v2.6.1 h1:XFkytnGvk/ZcY2qU0ql4E4h+ftBaGqkLO7tlZ4kRbr4=
+github.com/confluentinc/confluent-kafka-go/v2 v2.6.1/go.mod h1:hScqtFIGUI1wqHIgM3mjoqEou4VweGGGX7dMpcUKves=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4=
+github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=