diff --git a/.github/workflows/ci-merge-coverage.yaml b/.github/workflows/ci-merge-coverage.yaml new file mode 100644 index 000000000..d26d37bd4 --- /dev/null +++ b/.github/workflows/ci-merge-coverage.yaml @@ -0,0 +1,67 @@ +name: ci-merge-coverage + +on: + workflow_run: + workflows: [ci-test-ginkgo] + types: + - completed + +jobs: + merge-coverage-files: + name: Download and merge files + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Check if all required workflows completed successfully + id: check-workflows + run: | + workflows=("ci-test-ginkgo") + all_completed=true + + commit_sha=$(git rev-parse HEAD) + + for workflow in "${workflows[@]}"; do + conclusion=$(gh run list --workflow=$workflow --json conclusion,headSha,event,headBranch | jq -r --arg sha "$commit_sha" --arg event "pull_request" '.[] | select(.headSha == $sha and .event == $event) | .conclusion') + + if [[ -z "$conclusion" ]]; then + conclusion="pending" + fi + + if [[ "$conclusion" != "success" ]]; then + all_completed=false + fi + + echo "${workflow}_status=$conclusion" >> $GITHUB_ENV + done + + if [[ "$all_completed" == "true" ]]; then + echo "All workflows completed successfully for commit $commit_sha." + echo "all_succeeded=true" >> $GITHUB_ENV + else + echo "Not all workflows completed successfully for commit $commit_sha." + exit 1 + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/checkout@v3 + with: + submodules: true + + - uses: actions/setup-go@v5 + with: + go-version-file: 'KubeArmor/go.mod' + + - name: Download k8s coverage files from ci-test-ginkgo + if: ${{ env.ci-test-ginkgo_status == 'success' }} + uses: dawidd6/action-download-artifact@v6 + with: + workflow: ci-test-ginkgo.yml + name: coverage.* + path: KubeArmor/ + name_is_regexp: true + search_artifacts: true + + - uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/ci-test-ginkgo.yml b/.github/workflows/ci-test-ginkgo.yml index 5243c182f..553b5e68f 100644 --- a/.github/workflows/ci-test-ginkgo.yml +++ b/.github/workflows/ci-test-ginkgo.yml @@ -62,6 +62,8 @@ jobs: - name: Generate KubeArmor artifacts run: | + #set the $IS_COVERAGE env var to 'true' to build the kubearmor-test image for coverage calculation + export IS_COVERAGE=true GITHUB_SHA=$GITHUB_SHA ./KubeArmor/build/build_kubearmor.sh - name: Build Kubearmor-Operator @@ -82,8 +84,8 @@ jobs: - name: Run KubeArmor run: | if [[ ${{ matrix.runtime }} == "containerd" ]]; then - docker save kubearmor/kubearmor-init:latest | sudo k3s ctr images import - - docker save kubearmor/kubearmor:latest | sudo k3s ctr images import - + docker save kubearmor/kubearmor-test-init:latest | sudo k3s ctr images import - + docker save kubearmor/kubearmor-test:latest | sudo k3s ctr images import - docker save kubearmor/kubearmor-operator:latest | sudo k3s ctr images import - docker save kubearmor/kubearmor-snitch:latest | sudo k3s ctr images import - @@ -92,10 +94,10 @@ jobs: fi else if [ ${{ matrix.runtime }} == "crio" ]; then - docker save kubearmor/kubearmor-init:latest | sudo podman load - sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-init:latest - docker save kubearmor/kubearmor:latest | sudo podman load - sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor:latest + docker save kubearmor/kubearmor-test-init:latest | sudo podman load + sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-test-init:latest + docker save kubearmor/kubearmor-test:latest | sudo podman load + sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-test:latest docker save kubearmor/kubearmor-operator:latest | sudo podman load sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-operator:latest docker save kubearmor/kubearmor-snitch:latest | sudo podman load @@ -112,11 +114,11 @@ jobs: kubectl wait --for=condition=ready --timeout=5m -n kubearmor pod -l kubearmor-app=kubearmor-operator kubectl get pods -A if [[ ${{ steps.filter.outputs.controller }} == 'true' ]]; then - kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-test.yaml --dry-run=client -o json | \ + kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-coverage.yaml --dry-run=client -o json | \ jq '.spec.kubearmorControllerImage.imagePullPolicy = "Never"' | \ kubectl apply -f - else - kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-test.yaml + kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-coverage.yaml fi kubectl wait -n kubearmor --timeout=5m --for=jsonpath='{.status.phase}'=Running kubearmorconfigs/kubearmorconfig-test @@ -124,6 +126,47 @@ jobs: kubectl wait --timeout=1m --for=condition=ready pod -l kubearmor-app=kubearmor-controller -n kubearmor kubectl get pods -A + sleep 10 + DAEMONSET_NAME=$(kubectl get daemonset -n kubearmor -o jsonpath='{.items[0].metadata.name}') + echo "DaemonSet: $DAEMONSET_NAME" + + kubectl patch daemonset $DAEMONSET_NAME -n kubearmor --type='json' -p='[ + { + "op": "add", + "path": "/spec/template/spec/volumes/-", + "value": { + "name": "coverage-storage", + "hostPath": { + "path": "/coverage", + "type": "DirectoryOrCreate" + } + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/volumeMounts/-", + "value": { + "mountPath": "/coverage", + "name": "coverage-storage" + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/args/-", + "value": "-test.coverprofile=/coverage/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out" + } + ]' + + sleep 15 + + - name: Get KubeArmor POD info + run: | + DAEMONSET_NAME=$(kubectl get daemonset -n kubearmor -o jsonpath='{.items[0].metadata.name}') + LABEL_SELECTOR=$(kubectl get daemonset $DAEMONSET_NAME -n kubearmor -o jsonpath='{.spec.selector.matchLabels}' | jq -r 'to_entries[] | "\(.key)=\(.value)"' | paste -sd, -) + POD_NAME=$(kubectl get pods -n kubearmor -l "$LABEL_SELECTOR" -o jsonpath='{.items[*].metadata.name}') + echo "Pod: $POD_NAME" + echo "POD_NAME=$POD_NAME" >> $GITHUB_ENV + - name: Test KubeArmor using Ginkgo run: | go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo @@ -131,6 +174,26 @@ jobs: working-directory: ./tests/k8s_env timeout-minutes: 30 + - name: Kill KubeArmor prcoess in the pod + run: | + KUBEARMOR_PID=$(kubectl exec ${{ env.POD_NAME }} -n kubearmor -c kubearmor -- sh -c "ps aux | grep '[K]ubeArmor/kubearmor-test' | awk '{print \$1}'") + kubectl exec ${{ env.POD_NAME }} -n kubearmor -c kubearmor -- sh -c "kill -s SIGINT $KUBEARMOR_PID" + sleep 10 + env: + POD_NAME: ${{ env.POD_NAME }} + + - name: Extract coverage file + run: | + for i in {1..24}; do + if [ -f /coverage/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out ]; then + cp /coverage/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out + break + fi + sleep 5 + done + ls -l + working-directory: KubeArmor + - name: Get karmor sysdump if: ${{ failure() }} run: | @@ -150,14 +213,15 @@ jobs: - name: Measure code coverage if: ${{ always() }} run: | - go install github.com/modocache/gover@latest - gover - go tool cover -func=gover.coverprofile + ls -l + go tool cover -func coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out working-directory: KubeArmor env: GOPATH: /home/runner/go - - uses: codecov/codecov-action@v3 + - name: Upload coverage file if: ${{ always() }} + uses: actions/upload-artifact@v4 with: - files: ./KubeArmor/gover.coverprofile + name: coverage-k8s-${{ matrix.os }}-${{ matrix.runtime }} + path: KubeArmor/coverage_k8s_${{ matrix.os }}_${{ matrix.runtime }}.out diff --git a/Dockerfile b/Dockerfile index 04729c768..b0ace30dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ### Builder -FROM golang:1.22-alpine3.20 as builder +FROM golang:1.22-alpine3.20 AS builder RUN apk --no-cache update RUN apk add --no-cache git clang llvm make gcc protobuf @@ -38,9 +38,15 @@ COPY ./KubeArmor/BPF . RUN make +### Builder test + +FROM builder AS builder-test +WORKDIR /usr/src/KubeArmor/KubeArmor +RUN CGO_ENABLED=0 go test -covermode=atomic -coverpkg=./... -c . -o kubearmor-test + ### Make executable image -FROM alpine:3.20 as kubearmor +FROM alpine:3.20 AS kubearmor RUN echo "@community http://dl-cdn.alpinelinux.org/alpine/edge/community" | tee -a /etc/apk/repositories @@ -53,6 +59,11 @@ COPY --from=builder /usr/src/KubeArmor/KubeArmor/templates/* /KubeArmor/template ENTRYPOINT ["/KubeArmor/kubearmor"] +FROM kubearmor AS kubearmor-test +COPY --from=builder-test /usr/src/KubeArmor/KubeArmor/kubearmor-test /KubeArmor/kubearmor-test + +ENTRYPOINT ["/KubeArmor/kubearmor-test"] + ### TODO ### ### build apparmor_parser binary @@ -65,7 +76,7 @@ ENTRYPOINT ["/KubeArmor/kubearmor"] ### Make UBI-based executable image -FROM redhat/ubi9-minimal as kubearmor-ubi +FROM redhat/ubi9-minimal AS kubearmor-ubi ARG VERSION=latest ENV KUBEARMOR_UBI=true @@ -100,4 +111,35 @@ RUN setcap "cap_sys_admin=ep cap_sys_ptrace=ep cap_ipc_lock=ep cap_sys_resource= USER 1000 ENTRYPOINT ["/KubeArmor/kubearmor"] +### Make UBI-based test executable image for coverage calculation +FROM redhat/ubi9-minimal AS kubearmor-ubi-test + +ARG VERSION=latest +ENV KUBEARMOR_UBI=true +LABEL name="kubearmor" \ + vendor="Accuknox" \ + version=${VERSION} \ + release=${VERSION} \ + summary="kubearmor container image based on redhat ubi" \ + description="KubeArmor is a cloud-native runtime security enforcement system that restricts the behavior \ + (such as process execution, file access, and networking operations) of pods, containers, and nodes (VMs) \ + at the system level." + +RUN microdnf -y update && \ + microdnf -y install --nodocs --setopt=install_weak_deps=0 --setopt=keepcache=0 shadow-utils procps libcap && \ + microdnf clean all + +RUN groupadd --gid 1000 default \ + && useradd --uid 1000 --gid default --shell /bin/bash --create-home default + +COPY LICENSE /licenses/license.txt +COPY --from=builder --chown=default:default /usr/src/KubeArmor/KubeArmor/kubearmor /KubeArmor/kubearmor +COPY --from=builder --chown=default:default /usr/src/KubeArmor/BPF/*.o /opt/kubearmor/BPF/ +COPY --from=builder --chown=default:default /usr/src/KubeArmor/KubeArmor/templates/* /KubeArmor/templates/ +COPY --from=builder-test --chown=default:default /usr/src/KubeArmor/KubeArmor/kubearmor-test /KubeArmor/kubearmor-test + +RUN setcap "cap_sys_admin=ep cap_sys_ptrace=ep cap_ipc_lock=ep cap_sys_resource=ep cap_dac_override=ep cap_dac_read_search=ep" /KubeArmor/kubearmor-test + +USER 1000 +ENTRYPOINT ["/KubeArmor/kubearmor-test"] \ No newline at end of file diff --git a/KubeArmor/build/build_kubearmor.sh b/KubeArmor/build/build_kubearmor.sh index 50529ccd6..3a58c1391 100755 --- a/KubeArmor/build/build_kubearmor.sh +++ b/KubeArmor/build/build_kubearmor.sh @@ -44,6 +44,46 @@ echo "[INFO] Removed existing $REPO images" unset LABEL [[ "$GITHUB_SHA" != "" ]] && LABEL="--label github_sha=$GITHUB_SHA" +# set the $IS_COVERAGE env var to 'true' to build the kubearmor-test image for coverage calculation +if [[ "$IS_COVERAGE" == "true" ]]; then + REPO="kubearmor/kubearmor-test" + + # build a kubearmor-test image + DTAG="-t $REPO:$VERSION" + echo "[INFO] Building $DTAG" + cd $ARMOR_HOME/..; docker build $DTAG -f Dockerfile --target kubearmor-test . $LABEL + + if [ $? != 0 ]; then + echo "[FAILED] Failed to build $REPO:$VERSION" + exit 1 + fi + echo "[PASSED] Built $REPO:$VERSION" + + # build a kubearmor-test-init image + DTAGINI="-t $REPO-init:$VERSION" + echo "[INFO] Building $DTAGINI" + cd $ARMOR_HOME/..; docker build $DTAGINI -f Dockerfile.init --build-arg VERSION=$VERSION --target kubearmor-init . $LABEL + + if [ $? != 0 ]; then + echo "[FAILED] Failed to build $REPO-init:$VERSION" + exit 1 + fi + echo "[PASSED] Built $REPO-init:$VERSION" + + # build kubearmor-ubi-test image + DTAGUBITEST="-t $UBIREPO-test:$VERSION" + echo "[INFO] Building $DTAGUBITEST" + cd $ARMOR_HOME/..; docker build $DTAGUBITEST -f Dockerfile --target kubearmor-ubi-test . $LABEL + + if [ $? != 0 ]; then + echo "[FAILED] Failed to build $DTAGUBITEST:$VERSION" + exit 1 + fi + echo "[PASSED] Built $DTAGUBITEST:$VERSION" + + exit 0 +fi + # build a kubearmor image DTAG="-t $REPO:$VERSION" echo "[INFO] Building $DTAG" diff --git a/KubeArmor/main_test.go b/KubeArmor/main_test.go index 36d69dcff..adf21aeb3 100644 --- a/KubeArmor/main_test.go +++ b/KubeArmor/main_test.go @@ -5,18 +5,19 @@ package main import ( "flag" + "fmt" "os" "strconv" "testing" ) var clusterPtr, gRPCPtr, logPathPtr *string -var enableKubeArmorPolicyPtr, enableKubeArmorHostPolicyPtr, enableKubeArmorVMPtr, coverageTestPtr *bool +var enableKubeArmorPolicyPtr, enableKubeArmorHostPolicyPtr, enableKubeArmorVMPtr, coverageTestPtr, enableK8sEnv, tlsEnabled *bool var defaultFilePosturePtr, defaultCapabilitiesPosturePtr, defaultNetworkPosturePtr, hostDefaultCapabilitiesPosturePtr, hostDefaultNetworkPosturePtr, hostDefaultFilePosturePtr *string func init() { // options (string) - clusterPtr = flag.String("cluster", "", "cluster name") + clusterPtr = flag.String("cluster", "default", "cluster name") // options (string) gRPCPtr = flag.String("gRPC", "32767", "gRPC port number") @@ -36,8 +37,11 @@ func init() { enableKubeArmorHostPolicyPtr = flag.Bool("enableKubeArmorHostPolicy", true, "enabling KubeArmorHostPolicy") enableKubeArmorVMPtr = flag.Bool("enableKubeArmorVm", false, "enabling KubeArmorVM") + enableK8sEnv = flag.Bool("k8s", true, "is k8s env?") + tlsEnabled = flag.Bool("tlsEnabled", false, "enable tls for secure connection?") + // options (boolean) - coverageTestPtr = flag.Bool("coverageTest", true, "enabling CoverageTest") + coverageTestPtr = flag.Bool("coverageTest", false, "enabling CoverageTest") } // TestMain - test to drive external testing coverage @@ -45,18 +49,22 @@ func TestMain(t *testing.T) { // Reset Test Flags before executing main flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) - // Set os args to set flags in main - os.Args = []string{"cmd", "--cluster", *clusterPtr, "--gRPC", *gRPCPtr, "--logPath", *logPathPtr, - "--defaultFilePosture", *defaultFilePosturePtr, - "--defaultNetworkPosture", *defaultNetworkPosturePtr, - "--defaultCapabilitiesPosture", *defaultCapabilitiesPosturePtr, - "--hostDefaultFilePosture", *hostDefaultFilePosturePtr, - "--hostDefaultNetworkPosture", *hostDefaultNetworkPosturePtr, - "--hostDefaultCapabilitiesPosture", *hostDefaultCapabilitiesPosturePtr, - "--enableKubeArmorPolicy", strconv.FormatBool(*enableKubeArmorPolicyPtr), - "--enableKubeArmorHostPolicy", strconv.FormatBool(*enableKubeArmorHostPolicyPtr), - "--enableKubeArmorVm", strconv.FormatBool(*enableKubeArmorVMPtr), - "--coverageTest", strconv.FormatBool(*coverageTestPtr)} + os.Args = []string{ + fmt.Sprintf("-cluster=%s", *clusterPtr), + fmt.Sprintf("-gRPC=%s", *gRPCPtr), + fmt.Sprintf("-logPath=%s", *logPathPtr), + fmt.Sprintf("-defaultFilePosture=%s", *defaultFilePosturePtr), + fmt.Sprintf("-defaultNetworkPosture=%s", *defaultNetworkPosturePtr), + fmt.Sprintf("-defaultCapabilitiesPosture=%s", *defaultCapabilitiesPosturePtr), + fmt.Sprintf("-hostDefaultFilePosture=%s", *hostDefaultFilePosturePtr), + fmt.Sprintf("-hostDefaultNetworkPosture=%s", *hostDefaultNetworkPosturePtr), + fmt.Sprintf("-hostDefaultCapabilitiesPosture=%s", *hostDefaultCapabilitiesPosturePtr), + fmt.Sprintf("-k8s=%s", strconv.FormatBool(*enableK8sEnv)), + fmt.Sprintf("-enableKubeArmorPolicy=%s", strconv.FormatBool(*enableKubeArmorPolicyPtr)), + fmt.Sprintf("-enableKubeArmorHostPolicy=%s", strconv.FormatBool(*enableKubeArmorHostPolicyPtr)), + fmt.Sprintf("-coverageTest=%s", strconv.FormatBool(*coverageTestPtr)), + fmt.Sprintf("-tlsEnabled=%s", strconv.FormatBool(*tlsEnabled)), + } t.Log("[INFO] Executed KubeArmor") main() diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..39c5d5e78 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,29 @@ +coverage: + status: + project: + default: + target: auto + threshold: 5% + base: auto + +ignore: + - "KubeArmor/enforcer/SELinuxEnforcer.go" + - "KubeArmor/enforcer/SELinuxEnforcer_test.go" + - "KubeArmor/enforcer/SELinuxHostProfile.go" + - "KubeArmor/kvmAgent" + - "KubeArmor/state" + +comment: + layout: "reach, diff, files" + behavior: default + require_changes: false + show_changes: true + show_critical_paths: true + +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no \ No newline at end of file diff --git a/pkg/KubeArmorOperator/config/samples/kubearmor-coverage.yaml b/pkg/KubeArmorOperator/config/samples/kubearmor-coverage.yaml new file mode 100644 index 000000000..ec67b6b96 --- /dev/null +++ b/pkg/KubeArmorOperator/config/samples/kubearmor-coverage.yaml @@ -0,0 +1,33 @@ +apiVersion: operator.kubearmor.com/v1 +kind: KubeArmorConfig +metadata: + labels: + app.kubernetes.io/name: kubearmorconfig + app.kubernetes.io/instance: kubearmorconfig-sample + app.kubernetes.io/part-of: kubearmoroperator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: kubearmoroperator + name: kubearmorconfig-test + namespace: kubearmor +spec: + defaultCapabilitiesPosture: block + defaultFilePosture: block + defaultNetworkPosture: block + defaultVisibility: process,file,network,capabilities + seccompEnabled: false + alertThrottling: false + maxAlertPerSec: 10 + throttleSec: 30 + kubearmorImage: + image: kubearmor/kubearmor-test:latest + imagePullPolicy: Never + kubearmorInitImage: + image: kubearmor/kubearmor-test-init:latest + imagePullPolicy: Never + kubearmorRelayImage: + image: kubearmor/kubearmor-relay-server:latest + imagePullPolicy: Always + kubearmorControllerImage: + image: kubearmor/kubearmor-controller:latest + imagePullPolicy: Always +