Use record_modifier to fix encoding for graylog

fluent · Feb 28, 2019 · f3d355b · f3d355b
1 parent 25cc7a1
commit f3d355b
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 3 deletions.
diff --git a/docker-image/v1.3/debian-graylog/Gemfile b/docker-image/v1.3/debian-graylog/Gemfile
@@ -10,6 +10,7 @@ gem "fluent-plugin-concat", "~> 2.3.0"
 gem "fluent-plugin-rewrite-tag-filter", "~> 2.1.0"
 gem "gelf", "3.0.0"
 gem "fluent-plugin-gelf-hs", "~> 1.0.7"
+gem "fluent-plugin-record-modifier", "~> 2.0.0"
 gem "fluent-plugin-kubernetes_metadata_filter", "~> 2.1.4"
 gem "ffi"
 gem "fluent-plugin-systemd", "~> 1.0.1"
diff --git a/docker-image/v1.3/debian-graylog/Gemfile.lock b/docker-image/v1.3/debian-graylog/Gemfile.lock
@@ -27,6 +27,8 @@ GEM
       lru_redux
     fluent-plugin-multi-format-parser (1.0.0)
       fluentd (>= 0.14.0, < 2)
+    fluent-plugin-record-modifier (2.0.1)
+      fluentd (>= 1.0, < 2)
     fluent-plugin-rewrite-tag-filter (2.1.1)
       fluent-config-regexp-type
       fluentd (>= 0.14.2, < 2)
@@ -57,7 +59,7 @@ GEM
     http_parser.rb (0.6.0)
     i18n (1.5.3)
       concurrent-ruby (~> 1.0)
-    json (2.1.0)
+    json (2.2.0)
     kubeclient (1.1.4)
       activesupport
       http (= 0.9.8)
@@ -102,11 +104,12 @@ DEPENDENCIES
   fluent-plugin-gelf-hs (~> 1.0.7)
   fluent-plugin-kubernetes_metadata_filter (~> 2.1.4)
   fluent-plugin-multi-format-parser (~> 1.0.0)
+  fluent-plugin-record-modifier (~> 2.0.0)
   fluent-plugin-rewrite-tag-filter (~> 2.1.0)
   fluent-plugin-systemd (~> 1.0.1)
   fluentd (= 1.3.3)
   gelf (= 3.0.0)
   oj (= 3.5.1)
 
 BUNDLED WITH
-   1.16.1
+   1.17.2
diff --git a/docker-image/v1.3/debian-graylog/conf/fluent.conf b/docker-image/v1.3/debian-graylog/conf/fluent.conf
@@ -6,6 +6,12 @@
 @include kubernetes.conf
 @include conf.d/*.conf
 
+# The gelf plugin assumes input in utf-8
+<filter **>
+  @type record_modifier
+  char_encoding utf-8
+</filter>
+
 <match **>
    @type gelf
    @id out_graylog

diff --git a/templates/Gemfile.erb b/templates/Gemfile.erb
@@ -62,8 +62,10 @@ gem "fluent-plugin-gcs", "~> 0.3"
 gem "gelf", "3.0.0"
 <% if is_v1 %>
 gem "fluent-plugin-gelf-hs", "~> 1.0.7"
+gem "fluent-plugin-record-modifier", "~> 2.0.0"
 <% else %>
 gem "fluent-plugin-gelf-hs"
+gem "fluent-plugin-record-modifier", "< 1.0.0"
 <% end %>
 <% when "logzio" %>
 <% if is_v1 %>
@@ -93,7 +95,7 @@ gem "fluent-plugin-splunk-enterprise"
 <% if is_v1 %>
 gem "fluent-plugin-kubernetes_metadata_filter", "~> 2.1.4"
 <% else %>
-gem "fluent-plugin-kubernetes_metadata_filter"
+gem "fluent-plugin-kubernetes_metadata_filter", "< 1.0.0"
 <% end %>
 <% if !is_alpine %>
 gem "ffi"

diff --git a/templates/conf/fluent.conf.erb b/templates/conf/fluent.conf.erb
@@ -151,6 +151,12 @@
   num_threads 2
 </match>
 <% when "graylog"%>
+# The gelf plugin assumes input in utf-8
+<filter **>
+  @type record_modifier
+  char_encoding utf-8
+</filter>
+
 <match **>
    @type gelf
    @id out_graylog