diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 49b9f58a4b6..8f060dc48cf 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -68,6 +68,7 @@ CHANGELOG* /libbeat/processors/dns/ @elastic/sec-deployment-and-devices /libbeat/processors/registered_domain/ @elastic/sec-deployment-and-devices /libbeat/processors/syslog/ @elastic/sec-deployment-and-devices +/libbeat/processors/translate_ldap_attribute/ @elastic/sec-windows-platform /libbeat/processors/translate_sid/ @elastic/sec-windows-platform /libbeat/reader/syslog/ @elastic/sec-deployment-and-devices /libbeat/scripts @elastic/ingest-eng-prod diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d532363dc32..c5c56ef9925 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -112,6 +112,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Set timeout of 1 minute for FQDN requests {pull}37756[37756] - Fix issue where old data could be saved in the memory queue after acknowledgment, increasing memory use {pull}41356[41356] - Ensure Elasticsearch output can always recover from network errors {pull}40794[40794] +- Add `translate_ldap_attribute` processor. {pull}41472[41472] *Auditbeat* diff --git a/filebeat/tests/integration/translate_ldap_attribute_test.go b/filebeat/tests/integration/translate_ldap_attribute_test.go new file mode 100644 index 00000000000..e2b0f877efc --- /dev/null +++ b/filebeat/tests/integration/translate_ldap_attribute_test.go @@ -0,0 +1,216 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package integration + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "testing" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/image" + "github.com/docker/docker/client" + "github.com/docker/go-connections/nat" + "github.com/go-ldap/ldap/v3" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/tests/integration" + "github.com/elastic/elastic-agent-autodiscover/docker" +) + +const translateguidCfg = ` +filebeat.inputs: + - type: filestream + id: "test-translateguidCfg" + paths: + - %s + +queue.mem: + flush.min_events: 1 + flush.timeout: 0.1s + +path.home: %s + +output.file: + path: ${path.home} + filename: "output-file" + +logging: + metrics: + enabled: false + +processors: + - add_fields: + fields: + guid: '%s' + - translate_ldap_attribute: + field: fields.guid + target_field: fields.common_name + ldap_address: 'ldap://localhost:1389' + ldap_base_dn: 'dc=example,dc=org' + ldap_bind_user: 'cn=admin,dc=example,dc=org' + ldap_bind_password: 'adminpassword' + ldap_search_attribute: 'entryUUID' +` + +func TestTranslateGUIDWithLDAP(t *testing.T) { + startOpenldapContainer(t) + + var entryUUID string + require.Eventually(t, func() bool { + var err error + entryUUID, err = getLDAPUserEntryUUID() + return err == nil + }, 10*time.Second, time.Second) + + filebeat := integration.NewBeat( + t, + "filebeat", + "../../filebeat.test", + ) + tempDir := filebeat.TempDir() + + // 1. Generate the log file path + logFilePath := path.Join(tempDir, "log.log") + integration.GenerateLogFile(t, logFilePath, 1, false) + + // 2. Write configuration file and start Filebeat + filebeat.WriteConfigFile( + fmt.Sprintf(translateguidCfg, logFilePath, tempDir, entryUUID), + ) + filebeat.Start() + + var outputFile string + require.Eventually(t, func() bool { + outputFiles, err := filepath.Glob(path.Join(tempDir, "output-file-*.ndjson")) + if err != nil { + return false + } + if len(outputFiles) != 1 { + return false + } + outputFile = outputFiles[0] + return true + }, 10*time.Second, time.Second) + + // 3. Wait for the event with the expected translated guid + filebeat.WaitFileContains( + outputFile, + fmt.Sprintf(`"fields":{"guid":"%s","common_name":["User1","user01"]}`, entryUUID), + 10*time.Second, + ) +} + +func startOpenldapContainer(t *testing.T) { + ctx := context.Background() + c, err := docker.NewClient(client.DefaultDockerHost, nil, nil) + if err != nil { + t.Fatal(err) + } + + reader, err := c.ImagePull(ctx, "bitnami/openldap:2", image.PullOptions{}) + if err != nil { + t.Fatal(err) + } + if _, err = io.Copy(os.Stdout, reader); err != nil { + t.Fatal(err) + } + reader.Close() + + resp, err := c.ContainerCreate(ctx, + &container.Config{ + Image: "bitnami/openldap:2", + ExposedPorts: nat.PortSet{ + "1389/tcp": struct{}{}, + }, + Env: []string{ + "LDAP_URI=ldap://openldap:1389", + "LDAP_BASE=dc=example,dc=org", + "LDAP_BIND_DN=cn=admin,dc=example,dc=org", + "LDAP_BIND_PASSWORD=adminpassword", + }, + }, + &container.HostConfig{ + PortBindings: nat.PortMap{ + "1389/tcp": []nat.PortBinding{ + { + HostIP: "0.0.0.0", + HostPort: "1389", + }, + }, + }, + }, nil, nil, "") + if err != nil { + t.Fatal(err) + } + + if err := c.ContainerStart(ctx, resp.ID, container.StartOptions{}); err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + defer c.Close() + if err := c.ContainerRemove(ctx, resp.ID, container.RemoveOptions{RemoveVolumes: true, Force: true}); err != nil { + t.Error(err) + } + }) +} + +func getLDAPUserEntryUUID() (string, error) { + // Connect to the LDAP server + l, err := ldap.DialURL("ldap://localhost:1389") + if err != nil { + return "", fmt.Errorf("failed to connect to LDAP server: %w", err) + } + defer l.Close() + + err = l.Bind("cn=admin,dc=example,dc=org", "adminpassword") + if err != nil { + return "", fmt.Errorf("failed to bind to LDAP server: %w", err) + } + + searchRequest := ldap.NewSearchRequest( + "dc=example,dc=org", + ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 1, 0, false, + "(cn=User1)", []string{"entryUUID"}, nil, + ) + + sr, err := l.Search(searchRequest) + if err != nil { + return "", fmt.Errorf("failed to execute search: %w", err) + } + + // Process search results + if len(sr.Entries) == 0 { + return "", errors.New("no entries found for the specified username.") + } + entry := sr.Entries[0] + entryUUID := entry.GetAttributeValue("entryUUID") + if entryUUID == "" { + return "", errors.New("entryUUID is empty") + } + return entryUUID, nil +} diff --git a/libbeat/cmd/instance/imports_common.go b/libbeat/cmd/instance/imports_common.go index be4174c0ea0..eb33bc27fe3 100644 --- a/libbeat/cmd/instance/imports_common.go +++ b/libbeat/cmd/instance/imports_common.go @@ -43,6 +43,7 @@ import ( _ "github.com/elastic/beats/v7/libbeat/processors/registered_domain" _ "github.com/elastic/beats/v7/libbeat/processors/script" _ "github.com/elastic/beats/v7/libbeat/processors/syslog" + _ "github.com/elastic/beats/v7/libbeat/processors/translate_ldap_attribute" _ "github.com/elastic/beats/v7/libbeat/processors/translate_sid" _ "github.com/elastic/beats/v7/libbeat/processors/urldecode" _ "github.com/elastic/beats/v7/libbeat/publisher/includes" // Register publisher pipeline modules diff --git a/libbeat/docs/processors-list.asciidoc b/libbeat/docs/processors-list.asciidoc index 4105666049d..341875f9f96 100644 --- a/libbeat/docs/processors-list.asciidoc +++ b/libbeat/docs/processors-list.asciidoc @@ -131,6 +131,9 @@ endif::[] ifndef::no_timestamp_processor[] * <> endif::[] +ifndef::no_translate_ldap_attribute_processor[] +* <> +endif::[] ifndef::no_translate_sid_processor[] * <> endif::[] @@ -279,6 +282,9 @@ endif::[] ifndef::no_timestamp_processor[] include::{libbeat-processors-dir}/timestamp/docs/timestamp.asciidoc[] endif::[] +ifndef::no_translate_ldap_attribute_processor[] +include::{libbeat-processors-dir}/translate_ldap_attribute/docs/translate_ldap_attribute.asciidoc[] +endif::[] ifndef::no_translate_sid_processor[] include::{libbeat-processors-dir}/translate_sid/docs/translate_sid.asciidoc[] endif::[] diff --git a/libbeat/processors/translate_ldap_attribute/config.go b/libbeat/processors/translate_ldap_attribute/config.go new file mode 100644 index 00000000000..b6b46410e98 --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/config.go @@ -0,0 +1,45 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package translate_ldap_attribute + +import ( + "github.com/elastic/elastic-agent-libs/transport/tlscommon" +) + +type config struct { + Field string `config:"field" validate:"required"` + TargetField string `config:"target_field"` + LDAPAddress string `config:"ldap_address" validate:"required"` + LDAPBaseDN string `config:"ldap_base_dn" validate:"required"` + LDAPBindUser string `config:"ldap_bind_user"` + LDAPBindPassword string `config:"ldap_bind_password"` + LDAPSearchAttribute string `config:"ldap_search_attribute" validate:"required"` + LDAPMappedAttribute string `config:"ldap_mapped_attribute" validate:"required"` + LDAPSearchTimeLimit int `config:"ldap_search_time_limit"` + LDAPTLS *tlscommon.Config `config:"ldap_ssl"` + + IgnoreMissing bool `config:"ignore_missing"` + IgnoreFailure bool `config:"ignore_failure"` +} + +func defaultConfig() config { + return config{ + LDAPSearchAttribute: "objectGUID", + LDAPMappedAttribute: "cn", + LDAPSearchTimeLimit: 30} +} diff --git a/libbeat/processors/translate_ldap_attribute/doc.go b/libbeat/processors/translate_ldap_attribute/doc.go new file mode 100644 index 00000000000..70ceee7297d --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/doc.go @@ -0,0 +1,21 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package translate_ldap_attribute provides a Beat processor for converting +// LDAP attributes from one to another. It is typically used for converting Windows +// Global Unique Identifiers (GUIDs) to object names. +package translate_ldap_attribute diff --git a/libbeat/processors/translate_ldap_attribute/docs/translate_ldap_attribute.asciidoc b/libbeat/processors/translate_ldap_attribute/docs/translate_ldap_attribute.asciidoc new file mode 100644 index 00000000000..aff1125f43a --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/docs/translate_ldap_attribute.asciidoc @@ -0,0 +1,92 @@ +[[processor-translate-guid]] +=== Translate GUID + +++++ +translate_ldap_attribute +++++ + +The `translate_ldap_attribute` processor translates an LDAP attributes between eachother. +It is typically used to translate AD Global Unique Identifiers (GUID) +into their common names. + +Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes +refer to their GUID's rather than the object's name and these values +sometimes appear in logs. + +If the search attribute is invalid (malformed) or does not map to any object on the domain +then this will result in the processor returning an error unless `ignore_failure` +is set. + +The result of this operation is an array of values, given that a single attribute +can hold multiple values. + +Note: the search attribute is expected to map to a single object. If it doesn't, +no error will be returned, but only results of the first entry will be added +to the event. + +[source,yaml] +---- +processors: + - translate_ldap_attribute: + field: winlog.event_data.ObjectGuid + ldap_address: "ldap://" + ldap_base_dn: "dc=example,dc=com" + ignore_missing: true + ignore_failure: true +---- + +The `translate_ldap_attribute` processor has the following configuration settings: + +.Translate GUID options +[options="header"] +|====== +| Name | Required | Default | Description +| `field` | yes | | Source field containing a GUID. +| `target_field` | no | | Target field for the mapped attribute value. If not set it will be replaced in place. +| `ldap_address` | yes | | LDAP server address. eg: `ldap://ds.example.com:389` +| `ldap_base_dn` | yes | | LDAP base DN. eg: `dc=example,dc=com` +| `ldap_bind_user` | no | | LDAP user. +| `ldap_bind_password` | no | | LDAP password. +| `ldap_search_attribute` | yes | `objectGUID` | LDAP attribute to search by. +| `ldap_mapped_attribute` | yes | `cn` | LDAP attribute to map to. +| `ldap_search_time_limit` | no | 30 | LDAP search time limit in seconds. +| `ldap_ssl`* | no | 30 | LDAP TLS/SSL connection settings. +| `ignore_missing` | no | false | Ignore errors when the source field is missing. +| `ignore_failure` | no | false | Ignore all errors produced by the processor. +|====== + +* Also see <> for a full description of the `ldap_ssl` options. + +If the searches are slow or you expect a high amount of different key attributes to be found, +consider using a cache processor to speed processing: + + +[source,yaml] +------------------------------------------------------------------------------- +processors: + - cache: + backend: + memory: + id: ldapguids + get: + key_field: winlog.event_data.ObjectGuid + target_field: winlog.common_name + ignore_missing: true + - if: + not: + - has_fields: winlog.common_name + then: + - translate_ldap_attribute: + field: winlog.event_data.ObjectGuid + target_field: winlog.common_name + ldap_address: "ldap://" + ldap_base_dn: "dc=example,dc=com" + - cache: + backend: + memory: + id: ldapguids + capacity: 10000 + put: + key_field: winlog.event_data.ObjectGuid + value_field: winlog.common_name +------------------------------------------------------------------------------- \ No newline at end of file diff --git a/libbeat/processors/translate_ldap_attribute/ldap.go b/libbeat/processors/translate_ldap_attribute/ldap.go new file mode 100644 index 00000000000..f83200e6652 --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/ldap.go @@ -0,0 +1,139 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package translate_ldap_attribute + +import ( + "crypto/tls" + "fmt" + "sync" + + "github.com/go-ldap/ldap/v3" +) + +// ldapClient manages a single reusable LDAP connection +type ldapClient struct { + conn *ldap.Conn + mu sync.Mutex + *ldapConfig +} + +type ldapConfig struct { + address string + baseDN string + username string + password string + searchAttr string + mappedAttr string + searchTimeLimit int + tlsConfig *tls.Config +} + +// newLDAPClient initializes a new ldapClient with a single connection +func newLDAPClient(config *ldapConfig) (*ldapClient, error) { + client := &ldapClient{ldapConfig: config} + + // Establish initial connection + if err := client.connect(); err != nil { + return nil, err + } + + return client, nil +} + +// connect establishes a new connection to the LDAP server +func (client *ldapClient) connect() error { + client.mu.Lock() + defer client.mu.Unlock() + + // Connect with or without TLS based on configuration + var opts []ldap.DialOpt + if client.tlsConfig != nil { + opts = append(opts, ldap.DialWithTLSConfig(client.tlsConfig)) + } + conn, err := ldap.DialURL(client.address, opts...) + if err != nil { + return fmt.Errorf("failed to dial LDAP server: %w", err) + } + + if client.password != "" { + err = conn.Bind(client.username, client.password) + } else { + err = conn.UnauthenticatedBind(client.username) + } + + if err != nil { + conn.Close() + return fmt.Errorf("failed to bind to LDAP server: %w", err) + } + + client.conn = conn + return nil +} + +// reconnect checks the connection's health and reconnects if necessary +func (client *ldapClient) reconnect() error { + client.mu.Lock() + defer client.mu.Unlock() + + // Check if the connection is still alive + if client.conn.IsClosing() { + return client.connect() + } + return nil +} + +// findObjectBy searches for an object and returns its mapped values. +func (client *ldapClient) findObjectBy(searchBy string) ([]string, error) { + // Ensure the connection is alive or reconnect if necessary + if err := client.reconnect(); err != nil { + return nil, fmt.Errorf("failed to reconnect: %w", err) + } + + client.mu.Lock() + defer client.mu.Unlock() + + // Format the filter and perform the search + filter := fmt.Sprintf("(%s=%s)", client.searchAttr, searchBy) + searchRequest := ldap.NewSearchRequest( + client.baseDN, + ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 1, client.searchTimeLimit, false, + filter, []string{client.mappedAttr}, nil, + ) + + // Execute search + result, err := client.conn.Search(searchRequest) + if err != nil { + return nil, fmt.Errorf("search failed: %w", err) + } + if len(result.Entries) == 0 { + return nil, fmt.Errorf("no entries found for search attribute %s", searchBy) + } + + // Retrieve the CN attribute + cn := result.Entries[0].GetAttributeValues(client.mappedAttr) + return cn, nil +} + +// close closes the LDAP connection +func (client *ldapClient) close() { + client.mu.Lock() + defer client.mu.Unlock() + if client.conn != nil { + client.conn.Close() + } +} diff --git a/libbeat/processors/translate_ldap_attribute/translate_ldap_attribute.go b/libbeat/processors/translate_ldap_attribute/translate_ldap_attribute.go new file mode 100644 index 00000000000..dec72263cfd --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/translate_ldap_attribute.go @@ -0,0 +1,125 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package translate_ldap_attribute + +import ( + "errors" + "fmt" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/processors" + jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/transport/tlscommon" +) + +const logName = "processor.translate_ldap_attribute" + +var errInvalidType = errors.New("search attribute field value is not a string") + +func init() { + processors.RegisterPlugin("translate_ldap_attribute", New) + jsprocessor.RegisterPlugin("TranslateLDAPAttribute", New) +} + +type processor struct { + config + client *ldapClient + log *logp.Logger +} + +func New(cfg *conf.C) (beat.Processor, error) { + c := defaultConfig() + if err := cfg.Unpack(&c); err != nil { + return nil, fmt.Errorf("fail to unpack the translate_ldap_attribute configuration: %w", err) + } + + return newFromConfig(c) +} + +func newFromConfig(c config) (*processor, error) { + ldapConfig := &ldapConfig{ + address: c.LDAPAddress, + baseDN: c.LDAPBaseDN, + username: c.LDAPBindUser, + password: c.LDAPBindPassword, + searchAttr: c.LDAPSearchAttribute, + mappedAttr: c.LDAPMappedAttribute, + searchTimeLimit: c.LDAPSearchTimeLimit, + } + if c.LDAPTLS != nil { + tlsConfig, err := tlscommon.LoadTLSConfig(c.LDAPTLS) + if err != nil { + return nil, fmt.Errorf("could not load provided LDAP TLS configuration: %w", err) + } + ldapConfig.tlsConfig = tlsConfig.ToConfig() + } + client, err := newLDAPClient(ldapConfig) + if err != nil { + return nil, err + } + return &processor{ + config: c, + client: client, + log: logp.NewLogger(logName), + }, nil +} + +func (p *processor) String() string { + return fmt.Sprintf("translate_ldap_attribute=[field=%s, ldap_address=%s, ldap_base_dn=%s, ldap_bind_user=%s, ldap_search_attribute=%s, ldap_mapped_attribute=%s]", + p.Field, p.LDAPAddress, p.LDAPBaseDN, p.LDAPBindUser, p.LDAPSearchAttribute, p.LDAPMappedAttribute) +} + +func (p *processor) Run(event *beat.Event) (*beat.Event, error) { + err := p.translateLDAPAttr(event) + if err == nil || p.IgnoreFailure || (p.IgnoreMissing && errors.Is(err, mapstr.ErrKeyNotFound)) { + return event, nil + } + return event, err +} + +func (p *processor) translateLDAPAttr(event *beat.Event) error { + v, err := event.GetValue(p.Field) + if err != nil { + return err + } + + guidString, ok := v.(string) + if !ok { + return errInvalidType + } + + cn, err := p.client.findObjectBy(guidString) + if err != nil { + return err + } + + field := p.Field + if p.TargetField != "" { + field = p.TargetField + } + _, err = event.PutValue(field, cn) + return err +} + +func (p *processor) Close() error { + p.client.close() + return nil +}