-
Notifications
You must be signed in to change notification settings - Fork 4.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add asynchronous ACK handling to S3 and SQS inputs (#40699)
Modify SQS ingestion to listen for ACKs asynchronously so that input workers can keep reading new objects after a previous one has been published, instead of blocking on full upstream ingestion. This addresses the bottleneck where ingesting many small objects is slow as each one waits for a full ingestion round trip. With a default configuration, SQS queues with many small objects are now ingested up to 60x faster.
- Loading branch information
Showing
31 changed files
with
593 additions
and
712 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -23112,6 +23112,38 @@ Contents of probable licence file $GOMODCACHE/github.com/xdg-go/[email protected]/LIC | |
| of your accepting any such warranty or additional liability. | ||
|
|
||
|
|
||
| -------------------------------------------------------------------------------- | ||
| Dependency : github.com/zyedidia/generic | ||
| Version: v1.2.1 | ||
| Licence type (autodetected): MIT | ||
| -------------------------------------------------------------------------------- | ||
|
|
||
| Contents of probable licence file $GOMODCACHE/github.com/zyedidia/[email protected]/LICENSE: | ||
|
|
||
| MIT License | ||
|
|
||
| Copyright (c) 2021: Zachary Yedidia. | ||
|
|
||
| Permission is hereby granted, free of charge, to any person obtaining | ||
| a copy of this software and associated documentation files (the | ||
| "Software"), to deal in the Software without restriction, including | ||
| without limitation the rights to use, copy, modify, merge, publish, | ||
| distribute, sublicense, and/or sell copies of the Software, and to | ||
| permit persons to whom the Software is furnished to do so, subject to | ||
| the following conditions: | ||
|
|
||
| The above copyright notice and this permission notice shall be | ||
| included in all copies or substantial portions of the Software. | ||
|
|
||
| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | ||
| EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | ||
| MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. | ||
| IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY | ||
| CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, | ||
| TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE | ||
| SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. | ||
|
|
||
|
|
||
| -------------------------------------------------------------------------------- | ||
| Dependency : go.elastic.co/apm/module/apmelasticsearch/v2 | ||
| Version: v2.6.0 | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| // or more contributor license agreements. Licensed under the Elastic License; | ||
| // you may not use this file except in compliance with the Elastic License. | ||
|
|
||
| package awss3 | ||
|
|
||
| import ( | ||
| "github.com/zyedidia/generic/queue" | ||
|
|
||
| "github.com/elastic/beats/v7/libbeat/beat" | ||
| "github.com/elastic/beats/v7/libbeat/common/acker" | ||
| ) | ||
|
|
||
| type awsACKHandler struct { | ||
| pending *queue.Queue[pendingACK] | ||
| ackedCount int | ||
|
|
||
| pendingChan chan pendingACK | ||
| ackChan chan int | ||
| } | ||
|
|
||
| type pendingACK struct { | ||
| eventCount int | ||
| ackCallback func() | ||
| } | ||
|
|
||
| func newAWSACKHandler() *awsACKHandler { | ||
| handler := &awsACKHandler{ | ||
| pending: queue.New[pendingACK](), | ||
|
|
||
| // Channel buffer sizes are somewhat arbitrary: synchronous channels | ||
| // would be safe, but buffers slightly reduce scheduler overhead since | ||
| // the ack loop goroutine doesn't need to wake up as often. | ||
| // | ||
| // pendingChan receives one message each time an S3/SQS worker goroutine | ||
| // finishes processing an object. If it is full, workers will not be able | ||
| // to advance to the next object until the ack loop wakes up. | ||
| // | ||
| // ackChan receives approximately one message every time an acknowledged | ||
| // batch of events contains at least one event from this input. (Sometimes | ||
| // fewer if messages can be coalesced.) If it is full, acknowledgement | ||
| // notifications for inputs/queue will stall until the ack loop wakes up. | ||
| // (This is a much worse consequence than pendingChan, but ackChan also | ||
| // receives fewer messages than pendingChan by a factor of ~thousands, | ||
| // so in practice it's still low-impact.) | ||
| pendingChan: make(chan pendingACK, 10), | ||
| ackChan: make(chan int, 10), | ||
| } | ||
| go handler.run() | ||
| return handler | ||
| } | ||
|
|
||
| func (ah *awsACKHandler) Add(eventCount int, ackCallback func()) { | ||
| ah.pendingChan <- pendingACK{ | ||
| eventCount: eventCount, | ||
| ackCallback: ackCallback, | ||
| } | ||
| } | ||
|
|
||
| // Called when a worker is closing, to indicate to the ack handler that it | ||
| // should shut down as soon as the current pending list is acknowledged. | ||
| func (ah *awsACKHandler) Close() { | ||
| close(ah.pendingChan) | ||
| } | ||
|
|
||
| func (ah *awsACKHandler) pipelineEventListener() beat.EventListener { | ||
| return acker.TrackingCounter(func(_ int, total int) { | ||
| // Notify the ack handler goroutine | ||
| ah.ackChan <- total | ||
| }) | ||
| } | ||
|
|
||
| // Listener that handles both incoming metadata and ACK | ||
| // confirmations. | ||
| func (ah *awsACKHandler) run() { | ||
| for { | ||
| select { | ||
| case result, ok := <-ah.pendingChan: | ||
| if ok { | ||
| ah.pending.Enqueue(result) | ||
| } else { | ||
| // Channel is closed, reset so we don't receive any more values | ||
| ah.pendingChan = nil | ||
| } | ||
| case count := <-ah.ackChan: | ||
| ah.ackedCount += count | ||
| } | ||
|
|
||
| // Finalize any objects that are now completed | ||
| for !ah.pending.Empty() && ah.ackedCount >= ah.pending.Peek().eventCount { | ||
| result := ah.pending.Dequeue() | ||
| ah.ackedCount -= result.eventCount | ||
| // Run finalization asynchronously so we don't block the SQS worker | ||
| // or the queue by ignoring the ack handler's input channels. Ordering | ||
| // is no longer important at this point. | ||
| if result.ackCallback != nil { | ||
| go result.ackCallback() | ||
| } | ||
| } | ||
|
|
||
| // If the input is closed and all acks are completed, we're done | ||
| if ah.pending.Empty() && ah.pendingChan == nil { | ||
| return | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.