Ability to read additioal config files

Add the possibility to pass additional config files to fluent-bit, so the audit logs can be shipped to additional destinations, eg splunk

Dockerfile

@@ -6,9 +6,13 @@ COPY .git Makefile go.* *.go /work/
 COPY pkg/ /work/pkg/
 RUN make bin/audit-forwarder
 
-FROM fluent/fluent-bit:1.7.3-debug
+# Need to keep fluent-bit version below 1.7.5 for now,
+# due to bug https://github.com/fluent/fluent-bit/issues/3699
+FROM fluent/fluent-bit:1.7.4-debug
 
 COPY --from=builder /work/bin/audit-forwarder /fluent-bit/bin/
-COPY *.conf /fluent-bit/etc/
+COPY fluent-bit.conf /fluent-bit/etc/
+COPY parsers.conf /fluent-bit/etc/
+COPY null.conf /fluent-bit/etc/add/
 
 CMD ["/fluent-bit/bin/audit-forwarder"]

fluent-bit.conf

@@ -6,6 +6,7 @@
     Name                  tail
     Path                  ${AUDIT_LOG_PATH}
     DB                    /audit.db
+    Tag                   audit
     Parser                audit
     Read_from_Head        On
     Buffer_Chunk_Size     2MB
@@ -14,7 +15,7 @@
 
 [OUTPUT]
     Name                  forward
-    Match                 *
+    Match                 audit
     Host                  ${AUDIT_TAILER_HOST}
     Port                  ${AUDIT_TAILER_PORT}
     Require_ack_response  True
@@ -25,3 +26,5 @@
     tls.crt_file          ${TLS_CRT_FILE}
     tls.key_file          ${TLS_KEY_FILE}
     tls.vhost             ${TLS_VHOST}
+
+@INCLUDE add/*.conf

kind/audit/add/splunk.conf

@@ -0,0 +1,36 @@
+
+[FILTER]
+    Name            rewrite_tag
+    Match           audit
+    Rule            $kind Event tosplunk true
+
+[FILTER]
+    Name            nest
+    Match           tosplunk
+    Operation       nest
+    Wildcard        *
+    Nest_under      event
+
+[FILTER]
+    Name            record_modifier
+    Match           tosplunk
+    Record          host cluster-name
+    Record          sourcetype kube:apiserver:auditlog
+    Record          source apiserver-pod-name
+    Record          index REPLACE_WITH_SPLUNK_INDEX
+
+[OUTPUT]
+    Name            splunk
+    Match           tosplunk
+    Host            REPLACE_WITH_SPLUNK_HEC_ENDPOINT
+    Port            REPLACE_WITH_PORT
+    Splunk_Token    REPLACE_WITH_SPLUNK_HEC_TOKEN
+    TLS             On
+    TLS.Verify      On
+    Retry_Limit     False
+    Splunk_Send_Raw On
+
+[OUTPUT]
+    Name            stdout
+    Match           tosplunk
+    Format          json_lines

kind/konnectivity-agent.yaml

@@ -34,7 +34,7 @@ spec:
         - --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token
         command:
         - /proxy-agent
-        image: k8s.gcr.io/kas-network-proxy/proxy-agent:v0.0.12
+        image: k8s.gcr.io/kas-network-proxy/proxy-agent:v0.0.15
         imagePullPolicy: IfNotPresent
         livenessProbe:
           failureThreshold: 3

kind/kustomize-auditforwarder-splunk/kube-apiserver_patch.yaml

@@ -0,0 +1,32 @@
+- op: add
+  path: /spec/containers/1
+  value:
+    image: ghcr.io/metal-stack/audit-forwarder:pr-add-splunk
+    imagePullPolicy: Always
+    name: audit-forwarder
+    env:
+    - name: AUDIT_KUBECFG
+      value: "/kube.config"
+    - name: AUDIT_LOG_LEVEL
+      value: "info"
+    volumeMounts:
+    - mountPath: /auditlog
+      name: auditlog
+    - mountPath: /kube.config
+      name: kubeconfig
+    - mountPath: /fluent-bit/etc/add
+      name: add-config
+- op: add
+  path: /spec/volumes/0
+  value:
+    hostPath:
+      path: /etc/kubernetes/audit/kube.config
+      type: File
+    name: kubeconfig
+- op: add
+  path: /spec/volumes/0
+  value:
+    hostPath:
+      path: /etc/kubernetes/audit/add
+      type: Directory
+    name: add-config

kind/kustomize-auditforwarder-splunk/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+- kube-apiserver.yaml
+patches:
+- path: kube-apiserver_patch.yaml
+  target:
+    kind: Pod
+    name: kube-apiserver

kind/kustomize-auditforwarder/kube-apiserver_patch.yaml

@@ -1,7 +1,7 @@
 - op: add
   path: /spec/containers/1
   value:
-    image: ghcr.io/metal-stack/audit-forwarder:pr-rework-proxy
+    image: ghcr.io/metal-stack/audit-forwarder:pr-add-splunk
     imagePullPolicy: Always
     name: audit-forwarder
     env:

kind/kustomize-konnectivity/kube-apiserver_konnectivity_patch.yaml

@@ -20,7 +20,7 @@
     - --delete-existing-uds-file=true
     command:
     - /proxy-server
-    image: k8s.gcr.io/kas-network-proxy/proxy-server:v0.0.12
+    image: k8s.gcr.io/kas-network-proxy/proxy-server:v0.0.15
     imagePullPolicy: IfNotPresent
     livenessProbe:
       failureThreshold: 3

kind/make-audit-forwarder-splunk

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# First check if a forwarder has already been applied and get the kube-apiserver manifest if it isn't.
+if grep forwarder kind-etc-kubernetes/manifests/kube-apiserver.yaml >/dev/null; then
+    echo "Forwarder config already applied."
+    if [ ! -f kustomize-auditforwarder-splunk/kube-apiserver.yaml ]; then
+        echo "No saved kube-apiserver manifest exists, exiting."
+        exit
+    else
+        if grep forwarder kustomize-auditforwarder-splunk/kube-apiserver.yaml >/dev/null; then
+            echo "Saved config contains forwarder too, can not patch. Exiting."
+            exit
+        fi
+    fi
+else
+    echo "Getting kube-apiserver manifest."
+    cp kind-etc-kubernetes/manifests/kube-apiserver.yaml kustomize-auditforwarder-splunk/
+fi
+
+# Patch the generated kind kubeconfig with the apiserver URL valid from within the cluster.
+
+echo "Generating the in-cluster kubeconfig:"
+
+# Get the IP and port from the apiserver manifest:
+line=`grep kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint kustomize-auditforwarder-splunk/kube-apiserver.yaml`
+apiserver=${line##*kubeadm.kubernetes.io\/kube-apiserver.advertise-address.endpoint:?}
+
+sed "s+https://.*$+https://$apiserver+" kube.config >kind-etc-kubernetes/audit/kube.config
+
+echo "Patching and applying the kube-apiserver manifest:"
+kustomize build kustomize-auditforwarder-splunk >kind-etc-kubernetes/manifests/kube-apiserver.yaml

null.conf

@@ -0,0 +1 @@
+# An empty fluent-bit config file so that the @INCLUDE directive in fluent-bit.conf does not fail.