chore: upgrade Fluentd to v1.15.3

This also upgrades:
- Ruby from v2.7 to v3.1
- Alpine from 3.14 to 3.16
and some other packages.
See the Dockerfiles for details.

.github/workflows/pull_requests.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/[email protected]
         with:
-          ruby-version: '2.6'
+          ruby-version: '3.1'
       - name: Install bundler
         run: gem install bundler
       - name: Test fluent-plugin-datapoint
@@ -24,7 +24,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/[email protected]
         with:
-          ruby-version: '2.6'
+          ruby-version: '3.1'
       - name: Install bundler
         run: gem install bundler
       - name: Test fluent-plugin-enhance-k8s-metadata
@@ -36,7 +36,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/[email protected]
         with:
-          ruby-version: '2.6'
+          ruby-version: '3.1'
       - name: Install bundler
         run: gem install bundler
       - name: Test fluent-plugin-events
@@ -48,7 +48,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/[email protected]
         with:
-          ruby-version: '2.6'
+          ruby-version: '3.1'
       - name: Install bundler
         run: gem install bundler
       - name: Test fluent-plugin-kubernetes-metadata-filter
@@ -60,7 +60,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/[email protected]
         with:
-          ruby-version: '2.6'
+          ruby-version: '3.1'
       - name: Install bundler
         run: gem install bundler
       - name: Test fluent-plugin-kubernetes-sumologic
@@ -72,7 +72,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/[email protected]
         with:
-          ruby-version: '2.6'
+          ruby-version: '3.1'
       - name: Install bundler
         run: gem install bundler
       - name: Test fluent-plugin-prometheus-format
@@ -84,7 +84,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/[email protected]
         with:
-          ruby-version: '2.6'
+          ruby-version: '3.1'
       - name: Install bundler
         run: gem install bundler
       - name: Test fluent-plugin-protobuf

CONTRIBUTING.md

@@ -1,5 +1,17 @@
 # Contributing Guide
 
+## Testing your changes
+
+After making the necessary changes, run the following commands to test that your changes work correctly.
+
+```sh
+make test
+make build
+make image-test
+make build-alpine
+make image-test-alpine
+```
+
 ## Regenerating Gemfile
 
 Run the following command in order to generate `Gemfile.lock` files for the specific plugin:

Dockerfile

@@ -1,5 +1,6 @@
 ARG FLUENTD_ARCH
-FROM ruby:2.7.6-bullseye AS builder
+# Use the same ruby and debian version as is used in the target Fluentd image below.
+FROM ruby:3.1.3-bullseye AS builder
 
 # Dependencies
 RUN apt-get update \
@@ -14,22 +15,15 @@ RUN apt-get update \
         sudo \
         unzip
 
-# Update Ruby gems to prevent vulnerabilities
-RUN gem install \
-        bundler:2.3.4 \
-        date:2.0.1 \
-        rdoc:6.4.0 \
-        rexml:3.2.5
-
 # Fluentd plugin dependencies
 RUN gem install \
-        fluentd:1.14.6 \
-        concurrent-ruby:1.1.8 \
-        google-protobuf:3.19.2 \
+        fluentd:1.15.3 \
+        concurrent-ruby:1.1.10 \
+        google-protobuf:3.21.12 \
         lru_redux:1.1.0 \
         net-http-persistent:4.0.1 \
-        snappy:0.0.17 \
-        specific_install:0.3.5
+        snappy:0.3.0 \
+        specific_install:0.3.8
 
 # Use unreleased Kubeclient version with persistent HTTP connections.
 RUN gem specific_install https://github.com/ManageIQ/kubeclient --ref 220b8d7af52180f9a0f69cb73f0723d2618cf3ef
@@ -41,14 +35,20 @@ RUN gem install \
         # && gem install fluent-plugin-google-cloud \
         # && gem install fluent-plugin-azure-storage-append-blob
 
-# FluentD plugins from RubyGems
+# Install Fluentd plugins
 RUN gem install \
-        fluent-plugin-concat:2.4.0 \
-        fluent-plugin-prometheus:1.6.1 \
-        fluent-plugin-record-modifier:2.0.1 \
-        fluent-plugin-rewrite-tag-filter:2.2.0 \
+	# https://github.com/fluent-plugins-nursery/fluent-plugin-concat
+        fluent-plugin-concat:2.5.0 \
+        # https://github.com/fluent/fluent-plugin-prometheus
+        fluent-plugin-prometheus:2.0.3 \
+        # https://github.com/repeatedly/fluent-plugin-record-modifier
+        fluent-plugin-record-modifier:2.1.1 \
+        # https://github.com/fluent/fluent-plugin-rewrite-tag-filter
+        fluent-plugin-rewrite-tag-filter:2.4.0 \
+        # https://github.com/SumoLogic/fluentd-output-sumologic/
         fluent-plugin-sumologic_output:1.8.0 \
-        fluent-plugin-systemd:1.0.2
+        # https://github.com/fluent-plugin-systemd/fluent-plugin-systemd
+        fluent-plugin-systemd:1.0.5
 
 WORKDIR /sumologic-kubernetes-fluentd
 
@@ -99,7 +99,7 @@ RUN gem install \
 RUN rm -rf /usr/local/bundle/cache/* \
  && find /usr/local/bundle/ -name "*.o" | xargs rm
 
-FROM fluent/fluentd:v1.14.6-debian${FLUENTD_ARCH}-1.0
+FROM fluent/fluentd:v1.15.3-debian${FLUENTD_ARCH}-1.0
 
 USER root
 
@@ -117,18 +117,7 @@ RUN apt-get update \
  && gem update cgi \
  && gem cleanup \
  && rm -rf /var/lib/apt/lists/ \
- && rm -rf /var/lib/dpkg/info/ \
- && rm -rf /usr/local/lib/ruby/2.7.0/bundler/ \
- && rm /usr/local/lib/ruby/2.7.0/bundler.rb \
- && rm /usr/local/lib/ruby/gems/2.7.0/specifications/default/bundler-*.gemspec \
- && rm -rf /usr/local/lib/ruby/2.7.0/json/ \
- && rm /usr/local/lib/ruby/2.7.0/json.rb \
- && rm /usr/local/lib/ruby/gems/2.7.0/specifications/default/json-*.gemspec \
- && rm -rf /usr/local/lib/ruby/2.7.0/rdoc/ \
- && rm /usr/local/lib/ruby/2.7.0/rdoc.rb \
- && rm /usr/local/lib/ruby/gems/2.7.0/specifications/default/rdoc-*.gemspec \
- && rm -rf /usr/local/lib/ruby/2.7.0/rexml/ \
- && rm /usr/local/lib/ruby/gems/2.7.0/specifications/default/rexml-*.gemspec
+ && rm -rf /var/lib/dpkg/info/
 
 COPY --from=builder --chown=fluent:fluent /usr/local/bundle /usr/local/bundle
 COPY ./entrypoint.sh /bin/

Makefile

@@ -7,6 +7,7 @@ REPO_URL = $(ECR_URL)/$(IMAGE_NAME)
 PLUGINS = $(wildcard fluent-plugin*)
 BUILD_PLUGINS = $(patsubst fluent-plugin%, build-fluent-plugin%, $(PLUGINS))
 TEST_PLUGINS = $(patsubst fluent-plugin%, test-fluent-plugin%, $(PLUGINS))
+BUNDLE_UPDATE_ALL = $(patsubst fluent-plugin%, bundle-update-fluent-plugin%, $(PLUGINS))
 
 build:
 	DOCKER_BUILDKIT=1 docker build \
@@ -79,3 +80,10 @@ _build-%:
 	( cd $* && bundle install --no-deployment )
 
 ${BUILD_PLUGINS}: build-%: _build-%
+
+.PHONY: bundle-update
+bundle-update: ${BUNDLE_UPDATE_ALL}
+
+.PHONY: bundle-update-%
+bundle-update-%:
+	(cd $* && bundle update --bundler)

alpine.Dockerfile

@@ -1,4 +1,5 @@
-FROM ruby:2.7.6-alpine3.14 AS builder
+# Use the same alpine and ruby version as in the target image below.
+FROM ruby:3.1.3-alpine3.16 AS builder
 
 RUN apk update \
  && apk add \
@@ -11,27 +12,20 @@ RUN apk update \
 
 RUN echo 'gem: --no-document' >> /etc/gemrc
 
-# Update Ruby gems to prevent vulnerabilities
-RUN gem install \
-        bundler:2.3.4 \
-        cgi:0.3.1 \
-        rdoc:6.4.0 \
-        rexml:3.2.5
-
 # Fluentd plugin dependencies
+# Copied from https://github.com/fluent/fluentd-docker-image/blob/74c97b2117ed012ac832c72c3de475b81647ed29/v1.15/alpine/Dockerfile
 RUN gem install \
-        async-http:0.54.0 \
+        async-http:0.56.6 \
         bigdecimal:1.4.4 \
-        concurrent-ruby:1.1.8 \
-        ext_monitor:0.1.2 \
-        fluentd:1.14.6 \
-        google-protobuf:3.19.2 \
-        json:2.4.1 \
+        concurrent-ruby:1.1.10 \
+        fluentd:1.15.3 \
+        google-protobuf:3.21.12 \
+        # json:2.6.2 \ # Ruby base image already has 2.6.3
         lru_redux:1.1.0 \
         net-http-persistent:4.0.1 \
-        oj:3.10.18 \
-        snappy:0.0.17 \
-        specific_install:0.3.5
+        oj:3.13.22 \
+        snappy:0.3.0 \
+        specific_install:0.3.8
 
 # Use unreleased Kubeclient version with persistent HTTP connections.
 RUN gem specific_install https://github.com/ManageIQ/kubeclient --ref 220b8d7af52180f9a0f69cb73f0723d2618cf3ef
@@ -43,14 +37,20 @@ RUN gem install \
         # && gem install fluent-plugin-google-cloud \
         # && gem install fluent-plugin-azure-storage-append-blob
 
-# FluentD plugins from RubyGems
+# Install Fluentd plugins
 RUN gem install \
-        fluent-plugin-concat:2.4.0 \
-        fluent-plugin-prometheus:1.6.1 \
-        fluent-plugin-record-modifier:2.0.1 \
-        fluent-plugin-rewrite-tag-filter:2.2.0 \
+	# https://github.com/fluent-plugins-nursery/fluent-plugin-concat
+        fluent-plugin-concat:2.5.0 \
+        # https://github.com/fluent/fluent-plugin-prometheus
+        fluent-plugin-prometheus:2.0.3 \
+        # https://github.com/repeatedly/fluent-plugin-record-modifier
+        fluent-plugin-record-modifier:2.1.1 \
+        # https://github.com/fluent/fluent-plugin-rewrite-tag-filter
+        fluent-plugin-rewrite-tag-filter:2.4.0 \
+        # https://github.com/SumoLogic/fluentd-output-sumologic/
         fluent-plugin-sumologic_output:1.8.0 \
-        fluent-plugin-systemd:1.0.2
+        # https://github.com/fluent-plugin-systemd/fluent-plugin-systemd
+        fluent-plugin-systemd:1.0.5
 
 WORKDIR /sumologic-kubernetes-fluentd
 
@@ -98,7 +98,11 @@ RUN gem install \
         --local fluent-plugin-prometheus-format \
         --local fluent-plugin-protobuf
 
-FROM ruby:2.7.6-alpine3.14
+# Use ruby as base image because the official Fluentd alpine image is not built for linux/arm/v7 or linux/arm64/v8.
+# Use the same alpine and ruby version as the base image to prevent issues.
+# https://github.com/fluent/fluentd-docker-image/blob/1f228c28b0429b22fecff6eac44c801ee61f5118/v1.15/alpine/Dockerfile
+# https://pkgs.alpinelinux.org/packages?name=ruby&branch=v3.16&repo=&arch=&maintainer=
+FROM ruby:3.1.3-alpine3.16
 
 # 1. Update system packages.
 # 2. Install required system packages.
@@ -107,18 +111,7 @@ RUN apk update \
  && apk add --no-cache \
         ca-certificates \
         snappy-dev \
-        tini \
- && rm -rf /usr/local/lib/ruby/2.7.0/bundler/ \
- && rm /usr/local/lib/ruby/2.7.0/bundler.rb \
- && rm /usr/local/lib/ruby/gems/2.7.0/specifications/default/bundler-*.gemspec \
- && rm -rf /usr/local/lib/ruby/2.7.0/cgi/ \
- && rm /usr/local/lib/ruby/2.7.0/cgi.rb \
- && rm /usr/local/lib/ruby/gems/2.7.0/specifications/default/cgi-*.gemspec \
- && rm -rf /usr/local/lib/ruby/2.7.0/rdoc/ \
- && rm /usr/local/lib/ruby/2.7.0/rdoc.rb \
- && rm /usr/local/lib/ruby/gems/2.7.0/specifications/default/rdoc-*.gemspec \
- && rm -rf /usr/local/lib/ruby/2.7.0/rexml/ \
- && rm /usr/local/lib/ruby/gems/2.7.0/specifications/default/rexml-*.gemspec
+        tini
 
 RUN delgroup ping \
     && addgroup -S -g 999 fluent \

ci/build-push-multiplatform.sh

@@ -29,7 +29,7 @@ readonly arm_pid=$!
 docker buildx build \
     --push \
     --platform=linux/arm64/v8 \
-    --build-arg FLUENTD_ARCH="-arm64" \
+    --build-arg FLUENTD_ARCH="" \
     --build-arg BUILD_TAG="${BUILD_TAG}" \
     --tag "${REPO_URL}:${BUILD_TAG}-arm64" \
     . &

fluent-plugin-datapoint/.bundle/config

@@ -1,2 +1 @@
 ---
-BUNDLE_FROZEN: "true"

fluent-plugin-datapoint/Gemfile.lock

@@ -2,38 +2,38 @@ PATH
   remote: .
   specs:
     fluent-plugin-datapoint (2.0.0)
-      fluentd (= 1.14.6)
+      fluentd (= 1.15.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
     concurrent-ruby (1.1.10)
     cool.io (1.7.1)
-    fluentd (1.14.6)
+    fluentd (1.15.3)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.2.5, < 3.0.0)
+      serverengine (>= 2.3.0, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
     http_parser.rb (0.8.0)
-    msgpack (1.5.1)
-    power_assert (2.0.2)
+    msgpack (1.6.0)
+    power_assert (2.0.3)
     rake (13.0.6)
-    serverengine (2.2.5)
+    serverengine (2.3.1)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     strptime (0.2.5)
     test-unit (3.5.7)
       power_assert
-    tzinfo (2.0.4)
+    tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2022.1)
+    tzinfo-data (1.2022.7)
       tzinfo (>= 1.0.0)
     webrick (1.7.0)
     yajl-ruby (1.4.3)
@@ -48,4 +48,4 @@ DEPENDENCIES
   test-unit (~> 3.0)
 
 BUNDLED WITH
-   2.2.33
+   2.4.1

fluent-plugin-datapoint/fluent-plugin-datapoint.gemspec

@@ -18,5 +18,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
-  spec.add_runtime_dependency "fluentd", "= 1.14.6"
+  spec.add_runtime_dependency "fluentd", "= 1.15.3"
 end

fluent-plugin-enhance-k8s-metadata/.bundle/config

@@ -1,2 +1 @@
 ---
-BUNDLE_FROZEN: "true"