Update levoai-log-parser to latest released version

Signed-off-by: levobot <[email protected]>

levoai/levoai-log-parser/.helmignore

@@ -0,0 +1 @@
+tests/

levoai/levoai-log-parser/Chart.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+appVersion: 1.0.0
+description: Levo helm chart for Filebeat
+home: https://github.com/levoai/charts
+icon: https://helm.elastic.co/icons/beats.png
+maintainers:
+- email: [email protected]
+  name: Support
+name: levoai-log-parser
+sources:
+- https://github.com/elastic/helm-charts
+version: 1.0.0

levoai/levoai-log-parser/config/filebeat.yml

@@ -0,0 +1,29 @@
+filebeat.inputs:
+  - type: aws-s3
+    bucket_arn: ${AWS_BUCKET_ARN}
+    number_of_workers: 5
+    bucket_list_interval: 300s
+    access_key_id: ${AWS_ACCESS_KEY_ID}
+    secret_access_key: ${AWS_SECRET_ACCESS_KEY}
+    default_region: ${AWS_DEFAULT_REGION}
+    expand_event_list_from_field: Records
+    processors:
+      - add_fields:
+          target: "resource"
+          fields:
+            telemetry_sdk_type: "access-logs"
+            source: "incapsula"
+            codec: "cef"
+            service_name: "${APP_NAME}"
+            levo_env: "${ENV_NAME}"
+      - drop_event:
+          when:
+            not:
+              regexp:
+                aws.s3.object.key: "^imperva/cef/"
+
+output.logstash:
+  hosts: ["${LOGSTASH_HOST}:${LOGSTASH_BEATS_PORT}"]
+
+logging.level: '${LOG_LEVEL}'
+logging.selectors: ["*"]

levoai/levoai-log-parser/config/logstash.conf

@@ -0,0 +1,166 @@
+input {
+  beats {
+    port => "${LOGSTASH_BEATS_PORT}"
+  }
+}
+
+filter {
+
+  # Filter out metadata and configuration lines, keeping only CEF events
+  if [message] !~ /^CEF:/ {
+    drop {}
+  }
+
+  # Parse CEF fields and map them to eBPF trace format
+  grok {
+    match => {
+      "message" => [
+        # Match basic CEF fields
+        "CEF:%{NUMBER:cef_version}\|%{DATA:device_vendor}\|%{DATA:device_product}\|%{DATA:device_version}\|%{DATA:signature_id}\|%{DATA:name}\|%{NUMBER:severity}\|%{GREEDYDATA:cef_extension}"
+      ]
+    }
+  }
+
+  # Extract relevant fields from CEF extensions
+  kv {
+    source => "cef_extension"
+    field_split => " "
+    value_split => "="
+    target => "cef_data"
+    include_brackets => true
+  }
+
+  # Convert request URL to URI
+  mutate {
+    gsub => [
+      # Replace sourceServiceName portion in the request to keep only URI and query parameters
+      "[cef_data][request]", "^%{[cef_data][sourceServiceName]}", ""
+    ]
+  }
+
+  # Map CEF fields to eBPF trace format
+  # resource - source, service_name & levo_env are populated in filebeat
+  mutate {
+    add_field => {
+      "[http_scheme]" => "%{[cef_data][app]}"
+      "[request][headers][:authority]" => "%{[cef_data][sourceServiceName]}"
+      "[request][headers][:method]" => "%{[cef_data][requestMethod]}"
+      "[request][headers][:path]" => "%{[cef_data][request]}"
+      "[request][headers][user-agent]" => "%{[cef_data][requestClientApplication]}"
+      "[request][body]" => ""
+      "[request][truncated]" => "False"
+      "[response][headers][:status]" => "%{[cef_data][cn1]}"
+      "[response][body]" => ""
+      "[response][truncated]" => "False"
+      "[resource][host_name]" => "%{[cef_data][sourceServiceName]}"
+      "[resource][telemetry_sdk_language]" => "ebpf"
+      "[resource][telemetry_sdk_name]" => "logstash"
+      "[resource][telemetry_sdk_version]" => "0.1.0"
+      "[duration_ns]" => "%{[cef_data][duration]}"
+      "[trace_id]" => "%{[cef_data][cs4]}"
+      "[span_kind]" => "SERVER"
+      "[local_net][ip]" => "%{[cef_data][sip]}"
+      "[local_net][port]" => "%{[cef_data][spt]}"
+      "[remote][ip]" => "%{[cef_data][dst]}"
+      "[remote][port]" => "%{[cef_data][dpt]}"
+    }
+  }
+
+  # Add query string to request URI if present
+  if [cef_data][qstr] {
+    mutate {
+      gsub => ["[cef_data][qstr]", "\\=", "="]
+    }
+    mutate {
+      update => { "[request][headers][:path]" => "%{[request][headers][:path]}?%{[cef_data][qstr]}" }
+    }
+  }
+
+  # Add additional headers to request and response if present
+  if [cef_data][additionalReqHeaders] {
+    mutate {
+      gsub => ["[cef_data][additionalReqHeaders]", "\\=", "="]
+    }
+    mutate {
+      add_field => {
+        "[request][additional_headers]" => "[%{[cef_data][additionalReqHeaders]}]"
+      }
+    }
+  }
+
+  if [cef_data][additionalResHeaders] {
+    mutate {
+      gsub => ["[cef_data][additionalResHeaders]", "\\=", "="]
+    }
+    mutate {
+      add_field => {
+        "[response][additional_headers]" => "[%{[cef_data][additionalResHeaders]}]"
+      }
+    }
+  }
+
+  # Add post body to request if present
+  if [cef_data][postbody] {
+    mutate {
+      gsub => ["[cef_data][postbody]", "\\=", "="]
+    }
+    mutate {
+      replace => {
+        "[request][body]" => "%{[cef_data][postbody]}"
+      }
+    }
+  }
+
+  # Convert milliseconds to nanoseconds
+  ruby {
+    code => "
+      if event.get('[cef_data][start]')
+        # Convert milliseconds to nanoseconds
+        request_time_ns = event.get('[cef_data][start]').to_i * 10**6
+        event.set('[request_time_ns]', request_time_ns)
+      end
+    "
+  }
+
+  # Convert fields to integer/numeric where applicable
+  mutate {
+    convert => {
+      "[request][truncated]" => "boolean"
+      "[response][headers][:status]" => "integer"
+      "[response][truncated]" => "boolean"
+      "[duration_ns]" => "integer"
+      "[request_time_ns]" => "integer"
+      "[local_net][port]" => "integer"
+      "[remote_net][port]" => "integer"
+    }
+  }
+
+  # Remove unnecessary fields
+  prune {
+    whitelist_names => [
+      "^http_scheme",
+      "^request",
+      "^response",
+      "^resource",
+      "^duration_ns",
+      "^request_time_ns",
+      "^trace_id",
+      "^span_kind",
+      "^local_net",
+      "^remote_net"
+    ]
+  }
+
+}
+
+output {
+  http {
+    http_method => "post"
+    url => "${LEVO_SATELLITE_URL}/1.0/ebpf/traces"
+    format => "json_batch"
+    headers => {
+      "Content-Type" => "application/json"
+      "x-levo-organization-id" => "${LEVOAI_ORG_ID}"
+    }
+  }
+}

levoai/levoai-log-parser/config/logstash.yml

@@ -0,0 +1,2 @@
+api.http.host: 0.0.0.0
+xpack.monitoring.enabled: false

levoai/levoai-log-parser/templates/configmap-filebeat.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: levoai-filebeat-config
+  labels:
+    app: levoai-filebeat
+    app.kubernetes.io/name: levoai-filebeat-config
+    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+data:
+  filebeat.yml: |
+{{ .Files.Get "config/filebeat.yml" | indent 4 }}

levoai/levoai-log-parser/templates/configmap-logstash.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: levoai-logstash-config
+  labels:
+    app: levoai-logstash
+    app.kubernetes.io/name: levoai-logstash-config
+    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+data:
+  logstash.conf: |
+{{ .Files.Get "config/logstash.conf" | indent 4 }}
+  logstash.yml: |
+{{ .Files.Get "config/logstash.yml" | indent 4 }}

levoai/levoai-log-parser/templates/deployment-filebeat.yaml

@@ -0,0 +1,104 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: levoai-filebeat
+  labels:
+    app: levoai-filebeat
+    app.kubernetes.io/name: levoai-filebeat-deployment
+    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  annotations:
+    checksum/config: {{ include (print $.Template.BasePath "/configmap-filebeat.yaml") . | sha256sum }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: levoai-filebeat
+  template:
+    metadata:
+      labels:
+        app: levoai-filebeat
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/configmap-filebeat.yaml") . | sha256sum }}
+    spec:
+      containers:
+        - name: levoai-filebeat
+          image: "{{ .Values.filebeat.image.registry }}/{{ .Values.filebeat.image.repository }}:{{ .Values.filebeat.image.tag }}"
+          args:
+            - "-e"
+            - "-c"
+            - "/usr/share/filebeat/filebeat.yml"
+            - "-E"
+            - "http.enabled=true"
+          imagePullPolicy: "IfNotPresent"
+          env:
+            - name: LOG_LEVEL
+              value: {{ .Values.filebeat.logLevel | default .Values.global.logLevel | quote }}
+            - name: LOGSTASH_HOST
+              value: {{ .Values.global.logstashHost }}
+            - name: LOGSTASH_BEATS_PORT
+              value: {{ .Values.global.logstashBeatsPort | quote }}
+            - name: AWS_BUCKET_ARN
+              valueFrom:
+                secretKeyRef:
+                  name: aws-credentials
+                  key: aws-bucket-arn
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: aws-credentials
+                  key: aws-access-key-id
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: aws-credentials
+                  key: aws-secret-access-key
+            - name: AWS_DEFAULT_REGION
+              valueFrom:
+                secretKeyRef:
+                  name: aws-credentials
+                  key: aws-default-region
+            - name: APP_NAME
+              value: {{ .Values.filebeat.appName }}
+            - name: ENV_NAME
+              value: {{ .Values.filebeat.envName }}
+          volumeMounts:
+            - name: filebeat-config
+              mountPath: /usr/share/filebeat/filebeat.yml
+              subPath: filebeat.yml
+            - name: varlog
+              mountPath: /var/log
+          livenessProbe:
+            exec:
+              command:
+                - sh
+                - -c
+                - |
+                  #!/usr/bin/env bash -e
+                  curl --fail 127.0.0.1:5066
+            failureThreshold: 3
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+          readinessProbe:
+            exec:
+              command:
+                - sh
+                - -c
+                - |
+                  #!/usr/bin/env bash -e
+                  filebeat test output
+            failureThreshold: 3
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+      volumes:
+        - name: filebeat-config
+          configMap:
+            name: levoai-filebeat-config
+        - name: varlog
+          hostPath:
+            path: /var/log
+            type: Directory