[MM-51801] Cloud native support (#34)

* [MM-52322] Support multiple APIs implementations (#23)

* JobService interface

* Bump build deps

* [MM-52323] Refactor docker implementation (#24)

* Refactor docker job service

* Add tests

* Build and release docker image (#25)

* [MM-52346] Kubernetes implementation (#26)

* Setup k8s client

* Implement k8s API

* Remove StopJob

* Implement Init() API call

* Setup k8s CI

* Update sample config

* Add local k8s development doc

* Use human friendly prefix for job names

* Add support for passing custom tolerations

* Expose version info (#28)

* Implement unauthorized client error (#27)

* Update k8s client deps to latest

* Decouple public packages (#32)

* Enforce MaxConcurrentJobs limit (#33)

* Bump calls-recorder

* Update dev docs

* Explicitly set image pulling policy

* Support fetching dev builds from calls-recorder-daily registry

.github/workflows/ci.yml

@@ -2,6 +2,7 @@ on: [push]
 name: CI
 jobs:
   test:
+    name: "test"
     env:
       GOPATH: ${{ github.workspace }}
 
@@ -11,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [1.18.x]
+        go-version: [1.19.x]
 
     runs-on: ubuntu-latest
 
@@ -20,12 +21,38 @@ jobs:
       uses: actions/setup-go@v2
       with:
         go-version: ${{ matrix.go-version }}
-    - name: Checkout Code
+    - name: Checkout code
       uses: actions/checkout@v2
       with:
         path: ${{ env.GOPATH }}/src/github.com/${{ github.repository }}
-    - name: Execute Tests
+    - name: Execute tests
       run: |
         go mod download
         go mod verify
         make test
+
+  k8s:
+    name: "k8s"
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          path: ${{ github.workspace }}/${{ github.repository }}
+      - name: Start minikube
+        id: minikube
+        uses: medyagh/setup-minikube@latest
+      - name: Build image
+        run: eval $(minikube -p minikube docker-env) && make docker-build
+        working-directory: ${{ github.workspace }}/${{ github.repository }}
+      - name: Create pod
+        run: |
+          kubectl run calls-offloader --image=calls-offloader:dev-$(git rev-parse --short HEAD) \
+            --env="LOGGER_CONSOLELEVEL=debug" --env="LOGGER_ENABLEFILE=false" --env="JOBS_APITYPE=kubernetes"
+        working-directory: ${{ github.workspace }}/${{ github.repository }}
+      - name: Show logs
+        run: sleep 4s && kubectl logs calls-offloader
+      - name: Check pod is running
+        run: test $(kubectl get pods calls-offloader -o jsonpath='{.status.phase}') = "Running"

.github/workflows/golangci-lint.yml

@@ -7,7 +7,7 @@ jobs:
     name: lint
     strategy:
       matrix:
-        go-version: [1.18.x]
+        go-version: [1.19.x]
     runs-on: ubuntu-latest
     steps:
       - name: Install Go
@@ -19,7 +19,7 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v2
         with:
-          version: v1.50.1
+          version: v1.52.2
 
           # Optional: if set to true then the action will use pre-installed Go.
           skip-go-installation: true

Makefile

@@ -30,17 +30,19 @@ CONFIG_APP_CODE         += ./cmd/offloader
 ## Docker Variables
 # Docker executable
 DOCKER                  := $(shell which docker)
+# Dockerfile's location
+DOCKER_FILE             += ./build/Dockerfile
 # Docker options to inherit for all docker run commands
-DOCKER_OPTS             += --rm --platform "linux/amd64"
+DOCKER_OPTS             += --rm -u $$(id -u):$$(id -g) --platform "linux/amd64"
 # Registry to upload images
 DOCKER_REGISTRY         ?= docker.io
 DOCKER_REGISTRY_REPO    ?= mattermost/${APP_NAME}-daily
 # Registry credentials
 DOCKER_USER             ?= user
 DOCKER_PASSWORD         ?= password
 ## Docker Images
-DOCKER_IMAGE_GO         += "golang:${GO_VERSION}@sha256:fa71e1447cb0241324162a6c51297206928d755b16142eceec7b809af55061e5"
-DOCKER_IMAGE_GOLINT     += "golangci/golangci-lint:v1.50.1@sha256:94388e00f07c64262b138a7508f857473e30fdf0f59d04b546a305fc12cb5961"
+DOCKER_IMAGE_GO         += "golang:${GO_VERSION}@sha256:dd9ad81920b63c7f9f18823d888d5fdcc7e7516086fd16654d07bc437f0e2427"
+DOCKER_IMAGE_GOLINT     += "golangci/golangci-lint:v1.52.2@sha256:5fa6a92ab28ca3421c88d2b6cd794c9759d05a999aceca73053d014aad41b9d3"
 DOCKER_IMAGE_DOCKERLINT += "hadolint/hadolint:v2.9.2@sha256:d355bd7df747a0f124f3b5e7b21e9dafd0cb19732a276f901f0fdee243ec1f3b"
 DOCKER_IMAGE_COSIGN     += "bitnami/cosign:1.8.0@sha256:8c2c61c546258fffff18b47bb82a65af6142007306b737129a7bd5429d53629a"
 DOCKER_IMAGE_GH_CLI     += "registry.internal.mattermost.com/images/build-ci:3.16.0@sha256:f6a229a9ababef3c483f237805ee4c3dbfb63f5de4fbbf58f4c4b6ed8fcd34b6"
@@ -125,12 +127,119 @@ build: go-build-docker ## to build
 .PHONY: release
 release: build github-release ## to build and release artifacts
 
+.PHONY: package
+package: docker-login docker-build docker-push ## to build, package and push the artifact to a container registry
+
+.PHONY: sign
+sign: docker-sign docker-verify ## to sign the artifact and perform verification
+
 .PHONY: lint
 lint: go-lint ## to lint
 
 .PHONY: test
 test: go-test ## to test
 
+.PHONY: docker-build
+docker-build: ## to build the docker image
+	@$(INFO) Performing Docker build ${APP_NAME}:${APP_VERSION}
+	$(AT)$(DOCKER) build \
+	--build-arg GO_IMAGE=${DOCKER_IMAGE_GO} \
+	-f ${DOCKER_FILE} . \
+	-t ${APP_NAME}:${APP_VERSION} || ${FAIL}
+	@$(OK) Performing Docker build ${APP_NAME}:${APP_VERSION}
+
+.PHONY: docker-push
+docker-push: ## to push the docker image
+	@$(INFO) Pushing to registry...
+	$(AT)$(DOCKER) tag ${APP_NAME}:${APP_VERSION} $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:${APP_VERSION} || ${FAIL}
+	$(AT)$(DOCKER) push $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:${APP_VERSION} || ${FAIL}
+# if we are on a latest semver APP_VERSION tag, also push latest
+ifneq ($(shell echo $(APP_VERSION) | egrep '^v([0-9]+\.){0,2}(\*|[0-9]+)'),)
+  ifeq ($(shell git tag -l --sort=v:refname | tail -n1),$(APP_VERSION))
+	$(AT)$(DOCKER) tag ${APP_NAME}:${APP_VERSION} $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:latest || ${FAIL}
+	$(AT)$(DOCKER) push $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:latest || ${FAIL}
+  endif
+endif
+	@$(OK) Pushing to registry $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:${APP_VERSION}
+
+.PHONY: docker-sign
+docker-sign: ## to sign the docker image
+	@$(INFO) Signing the docker image...
+	$(AT)echo "$${COSIGN_KEY}" > cosign.key && \
+	$(DOCKER) run ${DOCKER_OPTS} \
+	--entrypoint '/bin/sh' \
+        -v $(PWD):/app -w /app \
+	-e COSIGN_PASSWORD=${COSIGN_PASSWORD} \
+	-e HOME="/tmp" \
+    ${DOCKER_IMAGE_COSIGN} \
+	-c \
+	"echo Signing... && \
+	cosign login $(DOCKER_REGISTRY) -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} && \
+	cosign sign --key cosign.key $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:${APP_VERSION}" || ${FAIL}
+# if we are on a latest semver APP_VERSION tag, also sign latest tag
+ifneq ($(shell echo $(APP_VERSION) | egrep '^v([0-9]+\.){0,2}(\*|[0-9]+)'),)
+  ifeq ($(shell git tag -l --sort=v:refname | tail -n1),$(APP_VERSION))
+	$(DOCKER) run ${DOCKER_OPTS} \
+	--entrypoint '/bin/sh' \
+        -v $(PWD):/app -w /app \
+	-e COSIGN_PASSWORD=${COSIGN_PASSWORD} \
+	-e HOME="/tmp" \
+	${DOCKER_IMAGE_COSIGN} \
+	-c \
+	"echo Signing... && \
+	cosign login $(DOCKER_REGISTRY) -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} && \
+	cosign sign --key cosign.key $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:latest" || ${FAIL}
+  endif
+endif
+	$(AT)rm -f cosign.key || ${FAIL}
+	@$(OK) Signing the docker image: $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:${APP_VERSION}
+
+.PHONY: docker-verify
+docker-verify: ## to verify the docker image
+	@$(INFO) Verifying the published docker image...
+	$(AT)echo "$${COSIGN_PUBLIC_KEY}" > cosign_public.key && \
+	$(DOCKER) run ${DOCKER_OPTS} \
+	--entrypoint '/bin/sh' \
+	-v $(PWD):/app -w /app \
+	${DOCKER_IMAGE_COSIGN} \
+	-c \
+	"echo Verifying... && \
+	cosign verify --key cosign_public.key $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:${APP_VERSION}" || ${FAIL}
+# if we are on a latest semver APP_VERSION tag, also verify latest tag
+ifneq ($(shell echo $(APP_VERSION) | egrep '^v([0-9]+\.){0,2}(\*|[0-9]+)'),)
+  ifeq ($(shell git tag -l --sort=v:refname | tail -n1),$(APP_VERSION))
+	$(DOCKER) run ${DOCKER_OPTS} \
+	--entrypoint '/bin/sh' \
+	-v $(PWD):/app -w /app \
+	${DOCKER_IMAGE_COSIGN} \
+	-c \
+	"echo Verifying... && \
+	cosign verify --key cosign_public.key $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:latest" || ${FAIL}
+  endif
+endif
+	$(AT)rm -f cosign_public.key || ${FAIL}
+	@$(OK) Verifying the published docker image: $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:${APP_VERSION}
+
+.PHONY: docker-sbom
+docker-sbom: ## to print a sbom report
+	@$(INFO) Performing Docker sbom report...
+	$(AT)$(DOCKER) sbom ${APP_NAME}:${APP_VERSION} || ${FAIL}
+	@$(OK) Performing Docker sbom report
+
+.PHONY: docker-scan
+docker-scan: ## to print a vulnerability report
+	@$(INFO) Performing Docker scan report...
+	$(AT)$(DOCKER) scan ${APP_NAME}:${APP_VERSION} || ${FAIL}
+	@$(OK) Performing Docker scan report
+
+.PHONY: docker-lint
+docker-lint: ## to lint the Dockerfile
+	@$(INFO) Dockerfile linting...
+	$(AT)$(DOCKER) run -i ${DOCKER_OPTS} \
+	${DOCKER_IMAGE_DOCKERLINT} \
+	< ${DOCKER_FILE} || ${FAIL}
+	@$(OK) Dockerfile linting
+
 .PHONY: docker-login
 docker-login: ## to login to a container registry
 	@$(INFO) Dockerd login to container registry ${DOCKER_REGISTRY}...

build/Dockerfile

@@ -0,0 +1,27 @@
+# This dockerfile is used to build Mattermost rtcd
+# A multi stage build, with golang used as a builder
+# and gcr.io/distroless/static as runner
+ARG GO_IMAGE=golang:1.19.8@sha256:dd9ad81920b63c7f9f18823d888d5fdcc7e7516086fd16654d07bc437f0e2427
+# hadolint ignore=DL3006
+FROM ${GO_IMAGE} as builder
+
+#GO_BUILD_PLATFORMS holds the platforms that we will build the docker image against
+ARG GO_BUILD_PLATFORMS=linux-amd64
+
+# Setup directories structure and compile
+COPY . /src
+WORKDIR /src
+RUN make go-build
+
+# Shrink final image since we only need the rtcd binary
+# and use distroless container image as runner for security
+FROM gcr.io/distroless/static@sha256:d6fa9db9548b5772860fecddb11d84f9ebd7e0321c0cb3c02870402680cc315f as runner
+COPY --from=builder /src/dist/calls-offloader-linux-amd64 /opt/calls-offloader/bin/calls-offloader
+
+# Create and use unprivileged user to run the service
+COPY ./build/group ./build/passwd /etc/
+
+USER calls
+
+WORKDIR /opt/calls-offloader/bin
+ENTRYPOINT ["./calls-offloader"]

build/group

@@ -0,0 +1,2 @@
+root:x:0:
+calls:x:65532:

build/passwd

@@ -0,0 +1,2 @@
+root:x:0:0:root:/root:/sbin/nologin
+calls:x:65532:65532:calls:/home/calls:/sbin/nologin

config/config.sample.toml

@@ -27,7 +27,7 @@ security.session_cache.expiration_minutes = 1440
 data_source = "/tmp/calls-offloader-db"
 
 [jobs]
-# The underlying API used to create and manage jobs. At the moment, "docker" is allowed.
+# The underlying API used to create and manage jobs. Allowed values are "docker" and "kubernetes".
 api_type = "docker"
 # Maximum number of jobs allowed to be running at one time.
 max_concurrent_jobs = 2