Adds more descriptive log parsing failures, uses rlo_14 for handling … (

#44) * Adds more descriptive log parsing failures, uses rlo_14 for handling syslog messages, adapts timestamp to use it * Don't skip parse * Updates rlo_14 to published version
teragrep · Jun 15, 2023 · 7f727fb · 7f727fb
1 parent 4c65f8c
commit 7f727fb
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 45 deletions.
diff --git a/pom.xml b/pom.xml
@@ -45,7 +45,7 @@
     <dependency>
       <groupId>com.teragrep</groupId>
       <artifactId>rlo_13</artifactId>
-      <version>1.0.1</version>
+      <version>1.1.0</version>
     </dependency>
 
     <!-- RELP sending library -->
@@ -55,12 +55,11 @@
       <version>3.3.1</version>
     </dependency>
 
-    <!-- https://mvnrepository.com/artifact/com.cloudbees/syslog-java-client -->
-    <!-- Handles syslog formatting -->
+    <!-- RFC5424 syslog formatting library -->
     <dependency>
-      <groupId>com.cloudbees</groupId>
-      <artifactId>syslog-java-client</artifactId>
-      <version>1.1.7</version>
+      <groupId>com.teragrep</groupId>
+      <artifactId>rlo_14</artifactId>
+      <version>1.0.1</version>
     </dependency>
 
     <!-- https://mvnrepository.com/artifact/io.kubernetes/client-java -->

diff --git a/src/main/java/com/teragrep/k8s_01/K8SConsumer.java b/src/main/java/com/teragrep/k8s_01/K8SConsumer.java
@@ -17,14 +17,13 @@
 
 package com.teragrep.k8s_01;
 
-import com.cloudbees.syslog.Facility;
-import com.cloudbees.syslog.SDElement;
-import com.cloudbees.syslog.Severity;
-import com.cloudbees.syslog.SyslogMessage;
+import com.teragrep.rlo_14.Facility;
+import com.teragrep.rlo_14.SDElement;
+import com.teragrep.rlo_14.Severity;
+import com.teragrep.rlo_14.SyslogMessage;
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParseException;
-import com.google.gson.JsonSyntaxException;
 import com.teragrep.k8s_01.config.AppConfig;
 import com.teragrep.k8s_01.metadata.KubernetesMetadata;
 import com.teragrep.k8s_01.metadata.NamespaceMetadataContainer;
@@ -77,34 +76,32 @@ public void accept(FileRecord record) {
                         record.getEndOffset()
                 );
             }
+            String namespace = ContainerInfo.getNamespace(record.getFilename());
+            String podname = ContainerInfo.getPodname(record.getFilename());
+            String containerId = ContainerInfo.getContainerID(record.getFilename());
             KubernetesLogFilePOJO log;
             try {
-                // We want to read the kubernetes log event into a POJO, mostly for the timestamp
+                // We want to read the kubernetes log event into a POJO
                 log = gson.fromJson(new String(record.getRecord()), KubernetesLogFilePOJO.class);
-            } catch (JsonSyntaxException e) {
-                LOGGER.error(
-                        "[{}] Can't continue as syntax for the event was invalid:",
-                        uuid,
-                        e
-                );
-                LOGGER.debug(
+            } catch (JsonParseException e) {
+                LOGGER.trace(
                         "[{}] Invalid syntax message: {}",
                         uuid,
                         new String(record.getRecord())
                 );
-                throw new RuntimeException(e);
-            } catch (JsonParseException e) {
-                LOGGER.error(
-                        "[{}] Can't parse log event:",
-                        uuid,
-                        e
-                );
-                LOGGER.debug(
-                        "[{}] Invalid json message: {}",
-                        uuid,
-                        new String(record.getRecord())
+                throw new RuntimeException(
+                        String.format(
+                                "[%s] Event from pod <%s>/<%s> on container <%s> in file <%s/%s> offset <%s> can't be parsed properly: %s",
+                                uuid,
+                                namespace,
+                                podname,
+                                containerId,
+                                record.getPath(),
+                                record.getFilename(),
+                                record.getStartOffset(),
+                                e.getMessage()
+                        )
                 );
-                throw new RuntimeException(e);
             }
 
             // Log is in invalid format if timestamp is not available
@@ -116,8 +113,11 @@ public void accept(FileRecord record) {
                 );
                 throw new RuntimeException(
                     String.format(
-                        "[%s] Can't parse record properly from %s/%s at %s",
+                        "[%s] Didn't find expected values for event from pod <%s/%s> on container <%s> in file %s/%s at offset %s",
                         uuid,
+                        namespace,
+                        podname,
+                        containerId,
                         record.getPath(),
                         record.getFilename(),
                         record.getStartOffset()
@@ -131,8 +131,6 @@ public void accept(FileRecord record) {
                     log
             );
 
-            String namespace = ContainerInfo.getNamespace(record.getFilename());
-            String podname = ContainerInfo.getPodname(record.getFilename());
             NamespaceMetadataContainer namespaceMetadataContainer = cacheClient.getNamespace(namespace);
             PodMetadataContainer podMetadataContainer = cacheClient.getPod(namespace, podname);
             KubernetesMetadata kubernetesMetadata = new KubernetesMetadata(
@@ -142,7 +140,6 @@ public void accept(FileRecord record) {
                     appConfig.getKubernetes().getUrl()
             );
             JsonObject dockerMetadata = new JsonObject();
-            String containerId = ContainerInfo.getContainerID(record.getFilename());
             dockerMetadata.addProperty("container_id", containerId);
 
             // Handle hostname and appname, use fallback values when labels are empty or if label not found

diff --git a/src/main/java/com/teragrep/k8s_01/KubernetesLogFilePOJO.java b/src/main/java/com/teragrep/k8s_01/KubernetesLogFilePOJO.java
@@ -42,15 +42,8 @@ public String getTime() {
         return time;
     }
 
-    public Date getTimestamp() {
-        // 2023-03-30T14:37:39.776175466Z
-        DateTimeFormatter dateTimeFormatter = new DateTimeFormatterBuilder()
-                .appendPattern("yyyy-MM-dd'T'HH:mm:ss.")
-                .appendFraction(ChronoField.NANO_OF_SECOND, 1, 9, false)
-                .appendPattern("X")
-                .toFormatter();
-        LocalDateTime localDateTime = LocalDateTime.parse(time, dateTimeFormatter);
-        return Date.from(localDateTime.toInstant(ZoneOffset.ofHours(0))); // FIXME: The timestamp offset needs to be verified, we do not want to edit it?
+    public String getTimestamp() {
+        return time;
     }
 
     @Override

diff --git a/src/main/java/com/teragrep/k8s_01/RelpOutput.java b/src/main/java/com/teragrep/k8s_01/RelpOutput.java
@@ -17,7 +17,7 @@
 
 package com.teragrep.k8s_01;
 
-import com.cloudbees.syslog.SyslogMessage;
+import com.teragrep.rlo_14.SyslogMessage;
 import com.codahale.metrics.*;
 import com.teragrep.k8s_01.config.AppConfigRelp;
 import com.teragrep.rlp_01.RelpBatch;