Merge remote-tracking branch 'origin/main' into dinesh.gurumurthy/fix…

…-connector
open-telemetry · Feb 5, 2024 · 0113c84 · 0113c84
2 parents 7459120 + 292f291
commit 0113c84
Show file tree

Hide file tree

Showing 20 changed files with 190 additions and 116 deletions.
diff --git a/.github/workflows/telemetrygen.yml b/.github/workflows/telemetrygen.yml
@@ -5,6 +5,10 @@ on:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
   pull_request:
+env:
+  # Make sure to exit early if cache segment download times out after 2 minutes.
+  # We limit cache download as a whole to 5 minutes.
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2
 
 # Do not cancel this workflow on main. See https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/16616
 concurrency:
@@ -17,10 +21,30 @@ jobs:
     if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ~1.20.12
+          cache: false
+      - name: Cache Go
+        id: go-cache
+        timeout-minutes: 5
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+      - name: Build binaries
+        run: |
+          GOOS=linux GOARCH=ppc64le make telemetrygen
+          GOOS=linux GOARCH=arm64 make telemetrygen
+          GOOS=linux GOARCH=amd64 make telemetrygen
+          GOOS=linux GOARCH=s390x make telemetrygen
+          cp bin/telemetrygen_* cmd/telemetrygen/
       - name: Build telemetrygen
         uses: docker/build-push-action@v5
         with:
@@ -36,6 +60,19 @@ jobs:
       packages: write
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ~1.20.12
+          cache: false
+      - name: Cache Go
+        id: go-cache
+        timeout-minutes: 5
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
@@ -46,6 +83,13 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build binaries
+        run: |
+          GOOS=linux GOARCH=ppc64le make telemetrygen
+          GOOS=linux GOARCH=arm64 make telemetrygen
+          GOOS=linux GOARCH=amd64 make telemetrygen
+          GOOS=linux GOARCH=s390x make telemetrygen
+          cp bin/telemetrygen_* cmd/telemetrygen/
       - name: Push telemetrygen to Github packages
         uses: docker/build-push-action@v5
         with:
@@ -61,6 +105,19 @@ jobs:
       packages: write
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ~1.20.12
+          cache: false
+      - name: Cache Go
+        id: go-cache
+        timeout-minutes: 5
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
@@ -74,6 +131,13 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build binaries
+        run: |
+          GOOS=linux GOARCH=ppc64le make telemetrygen
+          GOOS=linux GOARCH=arm64 make telemetrygen
+          GOOS=linux GOARCH=amd64 make telemetrygen
+          GOOS=linux GOARCH=s390x make telemetrygen
+          cp bin/telemetrygen_* cmd/telemetrygen/
       - name: Push telemetrygen to Github packages
         uses: docker/build-push-action@v5
         with:

diff --git a/Makefile b/Makefile
@@ -250,7 +250,10 @@ docker-otelcontribcol:
 
 .PHONY: docker-telemetrygen
 docker-telemetrygen:
-	COMPONENT=telemetrygen $(MAKE) docker-component
+	GOOS=linux GOARCH=$(GOARCH) $(MAKE) telemetrygen
+	cp bin/telemetrygen_* cmd/telemetrygen/
+	cd cmd/telemetrygen && docker build --platform linux/$(GOARCH) -t telemetrygen:latest .
+	rm cmd/telemetrygen/telemetrygen_*
 
 .PHONY: generate
 generate: install-tools

diff --git a/cmd/telemetrygen/Dockerfile b/cmd/telemetrygen/Dockerfile
@@ -1,10 +1,3 @@
-FROM golang:1.21 AS build
-
-WORKDIR /src
-ADD . /src
-
-RUN CGO_ENABLED=0 go build -o /telemetrygen
-
 FROM alpine:latest as certs
 RUN apk --update add ca-certificates
 
@@ -13,7 +6,10 @@ FROM scratch
 ARG USER_UID=10001
 USER ${USER_UID}
 
+ARG TARGETOS
+ARG TARGETARCH
+
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-COPY --from=build /telemetrygen /telemetrygen
+COPY ./telemetrygen_${TARGETOS}_${TARGETARCH} /telemetrygen
 
 ENTRYPOINT ["/telemetrygen"]
diff --git a/cmd/telemetrygen/README.md b/cmd/telemetrygen/README.md
@@ -24,6 +24,16 @@ go install github.com/open-telemetry/opentelemetry-collector-contrib/cmd/telemet
 
 Check the [`go install` reference](https://go.dev/ref/mod#go-install) to install specific versions.
 
+### Docker build
+
+You can build locally the Docker image with:
+
+```bash
+make docker-telemetrygen
+```
+
+Using github actions, we also push a docker image on commit to main or on release to this [Github docker registry](https://github.com/open-telemetry/opentelemetry-collector-contrib/pkgs/container/opentelemetry-collector-contrib%2Ftelemetrygen).
+
 ## Running
 
 First, you'll need an OpenTelemetry Collector to receive the telemetry data. Follow the project's instructions for a detailed setting up guide. The following configuration file should be sufficient:

diff --git a/exporter/fileexporter/go.mod b/exporter/fileexporter/go.mod
@@ -3,7 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileex
 go 1.20
 
 require (
-	github.com/klauspost/compress v1.17.4
+	github.com/klauspost/compress v1.17.5
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.93.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.93.0
 	github.com/stretchr/testify v1.8.4

diff --git a/exporter/fileexporter/go.sum b/exporter/fileexporter/go.sum
diff --git a/exporter/sapmexporter/go.mod b/exporter/sapmexporter/go.mod
@@ -5,7 +5,7 @@ go 1.20
 require (
 	github.com/cenkalti/backoff/v4 v4.2.1
 	github.com/jaegertracing/jaeger v1.53.0
-	github.com/klauspost/compress v1.17.4
+	github.com/klauspost/compress v1.17.5
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.93.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk v0.93.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchperresourceattr v0.93.0

diff --git a/exporter/sapmexporter/go.sum b/exporter/sapmexporter/go.sum
diff --git a/pkg/stanza/fileconsumer/config.go b/pkg/stanza/fileconsumer/config.go
@@ -16,6 +16,7 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/decode"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/attrs"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/emit"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/fileset"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/fingerprint"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/header"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/reader"
@@ -163,16 +164,20 @@ func (c Config) buildManager(logger *zap.SugaredLogger, emit emit.Callback, spli
 		HeaderConfig:    hCfg,
 		DeleteAtEOF:     c.DeleteAfterRead,
 	}
-
+	knownFiles := make([]*fileset.Fileset[*reader.Metadata], 3)
+	for i := 0; i < len(knownFiles); i++ {
+		knownFiles[i] = fileset.New[*reader.Metadata](c.MaxConcurrentFiles / 2)
+	}
 	return &Manager{
 		SugaredLogger:     logger.With("component", "fileconsumer"),
 		readerFactory:     readerFactory,
 		fileMatcher:       fileMatcher,
 		pollInterval:      c.PollInterval,
 		maxBatchFiles:     c.MaxConcurrentFiles / 2,
 		maxBatches:        c.MaxBatches,
-		previousPollFiles: make([]*reader.Reader, 0, c.MaxConcurrentFiles/2),
-		knownFiles:        []*reader.Metadata{},
+		currentPollFiles:  fileset.New[*reader.Reader](c.MaxConcurrentFiles / 2),
+		previousPollFiles: fileset.New[*reader.Reader](c.MaxConcurrentFiles / 2),
+		knownFiles:        knownFiles,
 	}, nil
 }