From 69327a8d02dc0a37e038a05d664b7d2e5f406502 Mon Sep 17 00:00:00 2001 From: Gerald Combs Date: Thu, 27 Apr 2023 17:06:47 -0700 Subject: [PATCH 1/5] update(plugins/cloudtrail): Add dependencies to our Makefile Signed-off-by: Gerald Combs --- plugins/cloudtrail/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/cloudtrail/Makefile b/plugins/cloudtrail/Makefile index c4cf8499..eb12ed15 100755 --- a/plugins/cloudtrail/Makefile +++ b/plugins/cloudtrail/Makefile @@ -28,8 +28,8 @@ all: $(OUTPUT) clean: @rm -f $(OUTPUT) -$(OUTPUT): +$(OUTPUT): pkg/cloudtrail/*.go plugin/cloudtrail.go @$(GODEBUGFLAGS) $(GO) build -buildmode=c-shared -o $(OUTPUT) ./plugin readme: - @$(READMETOOL) -p ./$(OUTPUT) -f README.md \ No newline at end of file + @$(READMETOOL) -p ./$(OUTPUT) -f README.md From 88fcf492b15a30c18e11a3c7be251212c63129ca Mon Sep 17 00:00:00 2001 From: Gerald Combs Date: Fri, 28 Apr 2023 15:45:24 -0700 Subject: [PATCH 2/5] update(plugins/cloudtrail): Add an S3Interval option Add an "S3Interval" option, which limits log downloads to the specified time interval. Intervals can be a simple "relative time in the past to now", "an absolute timestamp until now", or a range of those two formats. The interval is applied at the far end using the StartAfter parameter and locally to filter log pathnames. In my informal testing here the time from capture start to first event is ~2s. Signed-off-by: Gerald Combs --- plugins/cloudtrail/README.md | 15 ++ plugins/cloudtrail/go.mod | 1 + plugins/cloudtrail/go.sum | 2 + plugins/cloudtrail/pkg/cloudtrail/config.go | 2 + plugins/cloudtrail/pkg/cloudtrail/interval.go | 63 ++++++++ plugins/cloudtrail/pkg/cloudtrail/source.go | 135 ++++++++++++++---- 6 files changed, 194 insertions(+), 24 deletions(-) create mode 100644 plugins/cloudtrail/pkg/cloudtrail/interval.go diff --git a/plugins/cloudtrail/README.md b/plugins/cloudtrail/README.md index ff18aa08..788246ef 100644 --- a/plugins/cloudtrail/README.md +++ b/plugins/cloudtrail/README.md @@ -106,10 +106,25 @@ The json object has the following properties: * `sqsDelete`: value is boolean. If true, then the plugin will delete sqs messages from the queue immediately after receiving them. (Default: true) * `s3DownloadConcurrency`: value is numeric. Controls the number of background goroutines used to download S3 files. (Default: 1) +* `S3Interval`: value is string. Download log files matching the specified time interval. Note that this matches log file *names*, not event timestamps. CloudTrail logs usually cover [the previous 5 minutes of activity](https://docs.aws.amazon.com/awscloudtrail/latest/userguide/get-and-view-cloudtrail-log-files.html). See *Time Intervals* below for possible formats. * `useS3SNS`: value is boolean. If true, then the plugin will expect SNS messages to originate from S3 instead of directly from Cloudtrail (Default: false) The init string can be the empty string, which is treated identically to `{}`. +### Time Intervals + +S3Interval values can be individual duration values or RFC 3339-style timestamps. In this case the interval will start at the specified time and end at the current time: +* `5d`: A simple duration relative to the current time. +* `2021-03-30T18:07:17Z`: A duration starting from the specified RFC 3339 formatted time. + +Simple durations must be a positive integer followed by `w` for weeks, `d` for days, `h` for hours, `m` for minutes, or `s` for seconds. +RFC 3339-style times must be formatted as a datestamp, the letter `T`, a timestamp with no fractional seconds, and the letter `Z`, e.g. `2021-03-30T18:07:17Z`. + +Values can also cover a range: +* `5d-2d`: A simple duration interval relative to the current time. +* `2023-04-05T06:00:00Z-2023-04-05T12:00:10Z`: An RFC 3339-style timestamp interval. +* `2023-04-05T06:00:00Z-5d`: A combination of an RFC 3339-style timestamp and a duration. + ### Plugin Open Params The format of the open params string is a uri-like string with one of the following forms: diff --git a/plugins/cloudtrail/go.mod b/plugins/cloudtrail/go.mod index 1721491d..00f43b6a 100644 --- a/plugins/cloudtrail/go.mod +++ b/plugins/cloudtrail/go.mod @@ -13,4 +13,5 @@ require ( github.com/aws/smithy-go v1.13.3 github.com/falcosecurity/plugin-sdk-go v0.7.1 github.com/valyala/fastjson v1.6.3 + github.com/xhit/go-str2duration/v2 v2.1.0 ) diff --git a/plugins/cloudtrail/go.sum b/plugins/cloudtrail/go.sum index 991a1512..a7b3e908 100644 --- a/plugins/cloudtrail/go.sum +++ b/plugins/cloudtrail/go.sum @@ -78,6 +78,8 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/plugins/cloudtrail/pkg/cloudtrail/config.go b/plugins/cloudtrail/pkg/cloudtrail/config.go index 867d47ba..16bb8eb1 100644 --- a/plugins/cloudtrail/pkg/cloudtrail/config.go +++ b/plugins/cloudtrail/pkg/cloudtrail/config.go @@ -19,6 +19,7 @@ package cloudtrail // Struct for plugin init config type PluginConfig struct { S3DownloadConcurrency int `json:"s3DownloadConcurrency" jsonschema:"title=S3 download concurrency,description=Controls the number of background goroutines used to download S3 files (Default: 32),default=32"` + S3Interval string `json:"s3Interval" jsonschema:"title=S3 log interval,description=Download log files over the specified interval (Default: 24h),default=24h"` SQSDelete bool `json:"sqsDelete" jsonschema:"title=Delete SQS messages,description=If true then the plugin will delete SQS messages from the queue immediately after receiving them (Default: true),default=true"` UseAsync bool `json:"useAsync" jsonschema:"title=Use async extraction,description=If true then async extraction optimization is enabled (Default: true),default=true"` UseS3SNS bool `json:"useS3SNS" jsonschema:"title=Use S3 SNS,description=If true then the plugin will expect SNS messages to originate from S3 instead of directly from Cloudtrail (Default: false),default=false"` @@ -29,6 +30,7 @@ type PluginConfig struct { func (p *PluginConfig) Reset() { p.SQSDelete = true p.S3DownloadConcurrency = 32 + p.S3Interval = "24h" p.UseAsync = true p.UseS3SNS = false p.AWS.Reset() diff --git a/plugins/cloudtrail/pkg/cloudtrail/interval.go b/plugins/cloudtrail/pkg/cloudtrail/interval.go new file mode 100644 index 00000000..d4cb0d92 --- /dev/null +++ b/plugins/cloudtrail/pkg/cloudtrail/interval.go @@ -0,0 +1,63 @@ +/* +Copyright (C) 2022 The Falco Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudtrail + +import ( + "regexp" + "time" + + "github.com/xhit/go-str2duration/v2" +) + +var RFC3339Simple = "2006-01-02T03:04:05Z" + +func parseEndpoint(endpoint string) (time.Time, error) { + utc := time.Now().UTC() + var endpointTime time.Time + var err error + + epRE := regexp.MustCompile(`^\d+[wdhms]$`) + if (epRE.MatchString(endpoint)) { + duration, err := str2duration.ParseDuration(endpoint) + if (err == nil) { + endpointTime = utc.Add(- duration) + } + } else { + endpointTime, err = time.Parse(RFC3339Simple, endpoint) + } + return endpointTime, err +} + +// endTime will be zero if no end interval was supplied. +func ParseInterval(interval string) (time.Time, time.Time, error) { + var startTime time.Time + var endTime time.Time + var err error + + // First, see if we have an interval. + intervalRE := regexp.MustCompile(`(.*)(\s*-\s*)(\d+[wdhms]$|\d{4}-\d{2}\d{2}[^Z]*Z)`) + matches := intervalRE.FindStringSubmatch(interval) + if matches != nil { + startTime, err = parseEndpoint(matches[1]) + if err == nil { + endTime, err = parseEndpoint(matches[3]) + } + } else { + startTime, err = parseEndpoint(interval) + } + return startTime, endTime, err +} diff --git a/plugins/cloudtrail/pkg/cloudtrail/source.go b/plugins/cloudtrail/pkg/cloudtrail/source.go index f2b559f6..8e7e2ffb 100644 --- a/plugins/cloudtrail/pkg/cloudtrail/source.go +++ b/plugins/cloudtrail/pkg/cloudtrail/source.go @@ -26,6 +26,7 @@ import ( "io/ioutil" "os" "path/filepath" + "regexp" "strings" "sync" "time" @@ -172,38 +173,124 @@ func (oCtx *PluginInstance) openS3(input string) error { return err } - // Fetch the list of keys + + type listOrigin struct { + prefix *string + startAfter *string + } + + var inputParams []listOrigin ctx := context.Background() - paginator := s3.NewListObjectsV2Paginator(oCtx.s3.client, &s3.ListObjectsV2Input{ - Bucket: &oCtx.s3.bucket, - Prefix: &prefix, - }) - for paginator.HasMorePages() { - page, err := paginator.NextPage(ctx) - if err != nil { - // Try friendlier error sources first. - var aErr smithy.APIError - if errors.As(err, &aErr) { - return fmt.Errorf(PluginName + " plugin error: %s: %s", aErr.ErrorCode(), aErr.ErrorMessage()) - } + startTime, endTime, err := ParseInterval(oCtx.config.S3Interval) + if err != nil { + return fmt.Errorf(PluginName + " invalid interval: \"%s\": %s", oCtx.config.S3Interval, err.Error()) + + } + if !endTime.IsZero() && endTime.Compare(startTime) > 0 { + return fmt.Errorf(PluginName + " start time %s must be less than end time %s", startTime.Format(RFC3339Simple), endTime.Format(RFC3339Simple)) + } + + // CloudTrail logs have the format + // bucket_name/prefix_name/AWSLogs/Account ID/CloudTrail/region/YYYY/MM/DD/AccountID_CloudTrail_RegionName_YYYYMMDDTHHmmZ_UniqueString.json.gz + // Reduce the number of pages we have to process using "StartAfter" parameters + // here, then trim individual filepaths below. + + intervalPrefix := prefix + startAfterSuffix := startTime.Format("2006/01/02/") - var oErr *smithy.OperationError - if errors.As(err, &oErr) { - return fmt.Errorf(PluginName + " plugin error: %s: %s", oErr.Service(), oErr.Unwrap()) + // For durations, carve out a special case for "Copy S3 URI" in the AWS console, which gives you + // bucket_name/prefix_name/AWSLogs// + awsLogsRE := regexp.MustCompile(`AWSLogs/\d+/?$`) + if awsLogsRE.MatchString(prefix) { + if (! strings.HasSuffix(intervalPrefix, "/")) { + intervalPrefix += "/" + } + intervalPrefix += "CloudTrail/" + } + + if strings.HasSuffix(intervalPrefix, "/CloudTrail/") { + delimiter := "/" + // Fetch the list of regions. + output, err := oCtx.s3.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &oCtx.s3.bucket, + Prefix: &intervalPrefix, + Delimiter: &delimiter, + }) + if err == nil { + for _, commonPrefix := range output.CommonPrefixes { + // startAfter doesn't have to be a real key. + startAfter := *commonPrefix.Prefix + startAfterSuffix + params := listOrigin {prefix: commonPrefix.Prefix, startAfter: &startAfter} + inputParams = append(inputParams, params) } + } + } + + filepathRE := regexp.MustCompile(`.*_CloudTrail_[^_]+_([^_]+)Z_`) + var startTS string + var endTS string - return fmt.Errorf(PluginName + " plugin error: failed to list objects: " + err.Error()) + if len(inputParams) > 0 { + startTS = startTime.Format("20060102T0304") + if !endTime.IsZero() { + endTS = endTime.Format("20060102T0304") } - for _, obj := range page.Contents { - path := obj.Key - isCompressed := strings.HasSuffix(*path, ".json.gz") - if filepath.Ext(*path) != ".json" && !isCompressed { - continue + } else { + // No region prefixes found, just use what we were given. + params := listOrigin {prefix: &prefix, startAfter: nil} + inputParams = append(inputParams, params) + } + + // Would it make sense to do this concurrently? + for _, params := range inputParams { + // Fetch the list of keys + paginator := s3.NewListObjectsV2Paginator(oCtx.s3.client, &s3.ListObjectsV2Input{ + Bucket: &oCtx.s3.bucket, + Prefix: params.prefix, + StartAfter: params.startAfter, + }) + + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + // Try friendlier error sources first. + var aErr smithy.APIError + if errors.As(err, &aErr) { + return fmt.Errorf(PluginName + " plugin error: %s: %s", aErr.ErrorCode(), aErr.ErrorMessage()) + } + + var oErr *smithy.OperationError + if errors.As(err, &oErr) { + return fmt.Errorf(PluginName + " plugin error: %s: %s", oErr.Service(), oErr.Unwrap()) + } + + return fmt.Errorf(PluginName + " plugin error: failed to list objects: " + err.Error()) } + for _, obj := range page.Contents { + path := obj.Key + + if startTS != "" { + matches := filepathRE.FindStringSubmatch(*path) + if matches != nil { + pathTS := matches[1] + if pathTS < startTS { + continue + } + if endTS != "" && pathTS > endTS { + continue + } + } + } + + isCompressed := strings.HasSuffix(*path, ".json.gz") + if filepath.Ext(*path) != ".json" && !isCompressed { + continue + } - var fi fileInfo = fileInfo{name: *path, isCompressed: isCompressed} - oCtx.files = append(oCtx.files, fi) + var fi fileInfo = fileInfo{name: *path, isCompressed: isCompressed} + oCtx.files = append(oCtx.files, fi) + } } } From 02d7fa3d27003825785df491d7ee315922a1c014 Mon Sep 17 00:00:00 2001 From: Gerald Combs Date: Wed, 3 May 2023 10:04:46 -0700 Subject: [PATCH 3/5] update(plugins/cloudtrail): Fix a time comparison Time.Compare was added in go 1.20. Signed-off-by: Gerald Combs --- plugins/cloudtrail/pkg/cloudtrail/source.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/cloudtrail/pkg/cloudtrail/source.go b/plugins/cloudtrail/pkg/cloudtrail/source.go index 8e7e2ffb..829b8b41 100644 --- a/plugins/cloudtrail/pkg/cloudtrail/source.go +++ b/plugins/cloudtrail/pkg/cloudtrail/source.go @@ -187,9 +187,6 @@ func (oCtx *PluginInstance) openS3(input string) error { return fmt.Errorf(PluginName + " invalid interval: \"%s\": %s", oCtx.config.S3Interval, err.Error()) } - if !endTime.IsZero() && endTime.Compare(startTime) > 0 { - return fmt.Errorf(PluginName + " start time %s must be less than end time %s", startTime.Format(RFC3339Simple), endTime.Format(RFC3339Simple)) - } // CloudTrail logs have the format // bucket_name/prefix_name/AWSLogs/Account ID/CloudTrail/region/YYYY/MM/DD/AccountID_CloudTrail_RegionName_YYYYMMDDTHHmmZ_UniqueString.json.gz @@ -235,6 +232,9 @@ func (oCtx *PluginInstance) openS3(input string) error { startTS = startTime.Format("20060102T0304") if !endTime.IsZero() { endTS = endTime.Format("20060102T0304") + if endTS < startTS { + return fmt.Errorf(PluginName + " start time %s must be less than end time %s", startTime.Format(RFC3339Simple), endTime.Format(RFC3339Simple)) + } } } else { // No region prefixes found, just use what we were given. From 07d11f24e97bf6365079f866aea34655f95877f6 Mon Sep 17 00:00:00 2001 From: Gerald Combs Date: Wed, 10 May 2023 15:42:19 -0700 Subject: [PATCH 4/5] update(plugins/cloudtrail): Remove a dependency Remove our dependency on str2duration. Fix our interval regex. Signed-off-by: Gerald Combs --- plugins/cloudtrail/go.mod | 1 - plugins/cloudtrail/go.sum | 2 -- plugins/cloudtrail/pkg/cloudtrail/interval.go | 29 ++++++++++++++----- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/plugins/cloudtrail/go.mod b/plugins/cloudtrail/go.mod index 00f43b6a..1721491d 100644 --- a/plugins/cloudtrail/go.mod +++ b/plugins/cloudtrail/go.mod @@ -13,5 +13,4 @@ require ( github.com/aws/smithy-go v1.13.3 github.com/falcosecurity/plugin-sdk-go v0.7.1 github.com/valyala/fastjson v1.6.3 - github.com/xhit/go-str2duration/v2 v2.1.0 ) diff --git a/plugins/cloudtrail/go.sum b/plugins/cloudtrail/go.sum index a7b3e908..991a1512 100644 --- a/plugins/cloudtrail/go.sum +++ b/plugins/cloudtrail/go.sum @@ -78,8 +78,6 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= -github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/plugins/cloudtrail/pkg/cloudtrail/interval.go b/plugins/cloudtrail/pkg/cloudtrail/interval.go index d4cb0d92..72a30e3f 100644 --- a/plugins/cloudtrail/pkg/cloudtrail/interval.go +++ b/plugins/cloudtrail/pkg/cloudtrail/interval.go @@ -18,9 +18,8 @@ package cloudtrail import ( "regexp" + "strconv" "time" - - "github.com/xhit/go-str2duration/v2" ) var RFC3339Simple = "2006-01-02T03:04:05Z" @@ -30,10 +29,24 @@ func parseEndpoint(endpoint string) (time.Time, error) { var endpointTime time.Time var err error - epRE := regexp.MustCompile(`^\d+[wdhms]$`) - if (epRE.MatchString(endpoint)) { - duration, err := str2duration.ParseDuration(endpoint) - if (err == nil) { + durationRE := regexp.MustCompile(`^(\d+)([wdhms])$`) + matches := durationRE.FindStringSubmatch(endpoint) + if matches != nil { + durI, err := strconv.Atoi(matches[1]) + if err == nil { + duration := time.Duration(durI) + switch matches[2] { + case "w": + duration *= time.Hour * 24 * 7 + case "d": + duration *= time.Hour * 24 + case "h": + duration *= time.Hour + case "m": + duration *= time.Minute + case "s": + duration *= time.Second + } endpointTime = utc.Add(- duration) } } else { @@ -49,12 +62,12 @@ func ParseInterval(interval string) (time.Time, time.Time, error) { var err error // First, see if we have an interval. - intervalRE := regexp.MustCompile(`(.*)(\s*-\s*)(\d+[wdhms]$|\d{4}-\d{2}\d{2}[^Z]*Z)`) + intervalRE := regexp.MustCompile(`(.*)\s*-\s*(\d+[wdhms]|\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)$`) matches := intervalRE.FindStringSubmatch(interval) if matches != nil { startTime, err = parseEndpoint(matches[1]) if err == nil { - endTime, err = parseEndpoint(matches[3]) + endTime, err = parseEndpoint(matches[2]) } } else { startTime, err = parseEndpoint(interval) From 0da7894dfc3410de3ff00982c805972d620495ec Mon Sep 17 00:00:00 2001 From: Gerald Combs Date: Thu, 13 Jul 2023 13:47:29 -0700 Subject: [PATCH 5/5] update(plugins/cloudtrail): Make our default interval "" Make our default interval "" (which fetches all logs) instead of "24h". Signed-off-by: Gerald Combs --- plugins/cloudtrail/pkg/cloudtrail/config.go | 4 +-- plugins/cloudtrail/pkg/cloudtrail/interval.go | 2 +- plugins/cloudtrail/pkg/cloudtrail/source.go | 26 ++++++++++++------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/plugins/cloudtrail/pkg/cloudtrail/config.go b/plugins/cloudtrail/pkg/cloudtrail/config.go index 16bb8eb1..ab814cfc 100644 --- a/plugins/cloudtrail/pkg/cloudtrail/config.go +++ b/plugins/cloudtrail/pkg/cloudtrail/config.go @@ -19,7 +19,7 @@ package cloudtrail // Struct for plugin init config type PluginConfig struct { S3DownloadConcurrency int `json:"s3DownloadConcurrency" jsonschema:"title=S3 download concurrency,description=Controls the number of background goroutines used to download S3 files (Default: 32),default=32"` - S3Interval string `json:"s3Interval" jsonschema:"title=S3 log interval,description=Download log files over the specified interval (Default: 24h),default=24h"` + S3Interval string `json:"s3Interval" jsonschema:"title=S3 log interval,description=Download log files over the specified interval (Default: no interval),default="` SQSDelete bool `json:"sqsDelete" jsonschema:"title=Delete SQS messages,description=If true then the plugin will delete SQS messages from the queue immediately after receiving them (Default: true),default=true"` UseAsync bool `json:"useAsync" jsonschema:"title=Use async extraction,description=If true then async extraction optimization is enabled (Default: true),default=true"` UseS3SNS bool `json:"useS3SNS" jsonschema:"title=Use S3 SNS,description=If true then the plugin will expect SNS messages to originate from S3 instead of directly from Cloudtrail (Default: false),default=false"` @@ -30,7 +30,7 @@ type PluginConfig struct { func (p *PluginConfig) Reset() { p.SQSDelete = true p.S3DownloadConcurrency = 32 - p.S3Interval = "24h" + p.S3Interval = "" p.UseAsync = true p.UseS3SNS = false p.AWS.Reset() diff --git a/plugins/cloudtrail/pkg/cloudtrail/interval.go b/plugins/cloudtrail/pkg/cloudtrail/interval.go index 72a30e3f..2ff94436 100644 --- a/plugins/cloudtrail/pkg/cloudtrail/interval.go +++ b/plugins/cloudtrail/pkg/cloudtrail/interval.go @@ -69,7 +69,7 @@ func ParseInterval(interval string) (time.Time, time.Time, error) { if err == nil { endTime, err = parseEndpoint(matches[2]) } - } else { + } else if interval != "" { startTime, err = parseEndpoint(interval) } return startTime, endTime, err diff --git a/plugins/cloudtrail/pkg/cloudtrail/source.go b/plugins/cloudtrail/pkg/cloudtrail/source.go index 829b8b41..e0e0dc26 100644 --- a/plugins/cloudtrail/pkg/cloudtrail/source.go +++ b/plugins/cloudtrail/pkg/cloudtrail/source.go @@ -182,6 +182,7 @@ func (oCtx *PluginInstance) openS3(input string) error { var inputParams []listOrigin ctx := context.Background() + // XXX Make empty mean no startTime. startTime, endTime, err := ParseInterval(oCtx.config.S3Interval) if err != nil { return fmt.Errorf(PluginName + " invalid interval: \"%s\": %s", oCtx.config.S3Interval, err.Error()) @@ -194,7 +195,6 @@ func (oCtx *PluginInstance) openS3(input string) error { // here, then trim individual filepaths below. intervalPrefix := prefix - startAfterSuffix := startTime.Format("2006/01/02/") // For durations, carve out a special case for "Copy S3 URI" in the AWS console, which gives you // bucket_name/prefix_name/AWSLogs// @@ -216,9 +216,13 @@ func (oCtx *PluginInstance) openS3(input string) error { }) if err == nil { for _, commonPrefix := range output.CommonPrefixes { - // startAfter doesn't have to be a real key. - startAfter := *commonPrefix.Prefix + startAfterSuffix - params := listOrigin {prefix: commonPrefix.Prefix, startAfter: &startAfter} + params := listOrigin {prefix: commonPrefix.Prefix} + if !startTime.IsZero() { + // startAfter doesn't have to be a real key. + startAfterSuffix := startTime.Format("2006/01/02/") + startAfter := *commonPrefix.Prefix + startAfterSuffix + params.startAfter = &startAfter + } inputParams = append(inputParams, params) } } @@ -229,12 +233,14 @@ func (oCtx *PluginInstance) openS3(input string) error { var endTS string if len(inputParams) > 0 { - startTS = startTime.Format("20060102T0304") - if !endTime.IsZero() { - endTS = endTime.Format("20060102T0304") - if endTS < startTS { - return fmt.Errorf(PluginName + " start time %s must be less than end time %s", startTime.Format(RFC3339Simple), endTime.Format(RFC3339Simple)) - } + if !startTime.IsZero() { + startTS = startTime.Format("20060102T0304") + if !endTime.IsZero() { + endTS = endTime.Format("20060102T0304") + if endTS < startTS { + return fmt.Errorf(PluginName + " start time %s must be less than end time %s", startTime.Format(RFC3339Simple), endTime.Format(RFC3339Simple)) + } + } } } else { // No region prefixes found, just use what we were given.