From 3b3657a07a1f6622597baf66437e9e5051457ba8 Mon Sep 17 00:00:00 2001 From: Hong Quach Date: Wed, 3 May 2023 15:02:59 -0700 Subject: [PATCH] fix: add tail parser type to support containerd The `containerd` runtime generate logs as a non-JSON string. When switched to `containerd` runtime, `fluentd` will fail to parse any non-JSON log message and produce a large amount of parse error messages in its container logs. Here is an open issue at `fluentd` repo: https://github.com/fluent/fluentd-kubernetes-daemonset/issues/412 **docker** runtime (a valid JSON string) `{"log":"2023-05-02 20:17:16 +0000 [info]: #0 [filter_kube_metadata_host] stats - namespace_cache_size: 0, pod_cache_size: 0\n","stream":"stdout","time":"2023-05-02T20:17:16.666667387Z"}` **containerd** runtime (just a string) `2023-05-02T20:17:28.143532061Z stdout F 2023-05-02 20:17:28 +0000 [info]: #0 [filter_kube_metadata_host] stats - namespace_cache_size: 0, pod_cache_size: 0` Here is an example of a short entry from a `fluentd` container log. ``` 2023-05-02 19:51:40 +0000 [warn]: #0 [in_tail_fluentd_logs] pattern not matched: "2023-05-02T19:51:17.411234908Z stdout F \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\"" ``` --- .../container-insights-monitoring/fluentd/fluentd.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluentd/fluentd.yaml b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluentd/fluentd.yaml index 22a7d93..eb94e3c 100644 --- a/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluentd/fluentd.yaml +++ b/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluentd/fluentd.yaml @@ -72,8 +72,8 @@ data: tag * read_from_head true - @type json - time_format %Y-%m-%dT%H:%M:%S.%NZ + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format %Y-%m-%dT%H:%M:%S.%N%:z @@ -86,8 +86,8 @@ data: tag * read_from_head true - @type json - time_format %Y-%m-%dT%H:%M:%S.%NZ + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format %Y-%m-%dT%H:%M:%S.%N%:z