From 878b04093152e9a0cfbe7c6601bcf893eb519be3 Mon Sep 17 00:00:00 2001 From: Andrew Gizas Date: Wed, 18 Sep 2024 18:18:53 +0300 Subject: [PATCH] Disable kubeadm for filebeat (both filbeat.input and filbeat.autodiscover) (#40301) * disable kubeadm for filebeat autodiscover * Update filebeat/_meta/test/docs/01_playground/filebeat.yaml Co-authored-by: Tiago Queiroz * Update libbeat/processors/add_kubernetes_metadata/docs/add_kubernetes_metadata.asciidoc Co-authored-by: Tiago Queiroz * Update dev-tools/kubernetes/filebeat/Dockerfile.debug Co-authored-by: Tiago Queiroz * fixing docs and adding mericbeat references * fixing docs and adding mericbeat references * Update filebeat/_meta/config/filebeat.autodiscover.reference.yml.tmpl Co-authored-by: Tetiana Kravchenko --------- Co-authored-by: Tiago Queiroz Co-authored-by: Tetiana Kravchenko (cherry picked from commit 89cd8409975fc2667e9ed19d53d4d1d1e4402c55) # Conflicts: # metricbeat/module/kubernetes/util/kubernetes.go --- CHANGELOG.next.asciidoc | 1 + .../kubernetes/filebeat/Dockerfile.debug | 2 +- .../kubernetes/heartbeat/Dockerfile.debug | 2 +- .../kubernetes/metricbeat/Dockerfile.debug | 2 +- .../filebeat.autodiscover.reference.yml.tmpl | 24 ++++++++++++++++ .../test/docs/01_playground/filebeat.yaml | 28 +++++++++++++++---- filebeat/filebeat.reference.yml | 24 ++++++++++++++++ .../providers/kubernetes/config.go | 2 ++ .../autodiscover/providers/kubernetes/pod.go | 9 ++++++ .../providers/kubernetes/service.go | 5 ++++ .../add_kubernetes_metadata/config.go | 1 + .../docs/add_kubernetes_metadata.asciidoc | 1 + .../add_kubernetes_metadata/kubernetes.go | 9 ++++++ metricbeat/docs/modules/kubernetes.asciidoc | 14 ++++++++++ metricbeat/metricbeat.reference.yml | 14 ++++++++++ .../kubernetes/_meta/config.reference.yml | 14 ++++++++++ .../module/kubernetes/util/kubernetes.go | 24 ++++++++++++++++ x-pack/filebeat/filebeat.reference.yml | 24 ++++++++++++++++ x-pack/metricbeat/metricbeat.reference.yml | 14 ++++++++++ 19 files changed, 206 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d48fbb6368a..56f67c3e0c4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -301,6 +301,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Disable event normalization for netflow input {pull}40635[40635] - Allow attribute selection in the Active Directory entity analytics provider. {issue}40482[40482] {pull}40662[40662] - Improve error quality when CEL program does not correctly return an events array. {pull}40580[40580] +- Add `use_kubeadm` config option for filebeat (both filbeat.input and autodiscovery) in order to toggle kubeadm-config api requests {pull}40301[40301] *Auditbeat* diff --git a/dev-tools/kubernetes/filebeat/Dockerfile.debug b/dev-tools/kubernetes/filebeat/Dockerfile.debug index eed6c864b57..3dbbae95ac7 100644 --- a/dev-tools/kubernetes/filebeat/Dockerfile.debug +++ b/dev-tools/kubernetes/filebeat/Dockerfile.debug @@ -4,7 +4,7 @@ ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin ENV CGO_ENABLED=0 -RUN go install github.com/go-delve/delve/cmd/dlv@v1.21.2 +RUN go install github.com/go-delve/delve/cmd/dlv@latest COPY build/filebeat-debugger /usr/share/filebeat/filebeat-debugger diff --git a/dev-tools/kubernetes/heartbeat/Dockerfile.debug b/dev-tools/kubernetes/heartbeat/Dockerfile.debug index b3fbd0480bb..0802bb16783 100644 --- a/dev-tools/kubernetes/heartbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/heartbeat/Dockerfile.debug @@ -4,7 +4,7 @@ ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin ENV CGO_ENABLED=0 -RUN go install github.com/go-delve/delve/cmd/dlv@v1.21.2 +RUN go install github.com/go-delve/delve/cmd/dlv@latest COPY build/heartbeat-debugger /usr/share/heartbeat/heartbeat-debugger diff --git a/dev-tools/kubernetes/metricbeat/Dockerfile.debug b/dev-tools/kubernetes/metricbeat/Dockerfile.debug index 40c8ce35f88..1d9e4a85d16 100644 --- a/dev-tools/kubernetes/metricbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/metricbeat/Dockerfile.debug @@ -4,7 +4,7 @@ ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin ENV CGO_ENABLED=0 -RUN go install github.com/go-delve/delve/cmd/dlv@v1.21.2 +RUN go install github.com/go-delve/delve/cmd/dlv@latest COPY build/metricbeat-debugger /usr/share/metricbeat/metricbeat-debugger diff --git a/filebeat/_meta/config/filebeat.autodiscover.reference.yml.tmpl b/filebeat/_meta/config/filebeat.autodiscover.reference.yml.tmpl index 35875fecca8..a140db517a1 100644 --- a/filebeat/_meta/config/filebeat.autodiscover.reference.yml.tmpl +++ b/filebeat/_meta/config/filebeat.autodiscover.reference.yml.tmpl @@ -14,3 +14,27 @@ # - type: container # paths: # - /var/log/containers/*.log + +#Example: for kubernetes container logs autodiscovery +# filebeat.autodiscover: +# providers: +# - type: kubernetes +# node: ${NODE_NAME} +# hints.enabled: true +# # By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. +# use_kubeadm: true +# hints.default_config: +# type: filestream +# id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id} +# paths: +# - /var/log/containers/*-${data.kubernetes.container.id}.log +# parsers: +# - container: ~ +# prospector: +# scanner: +# fingerprint.enabled: true +# symlinks: true +# file_identity.fingerprint: ~ + +#By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. +# use_kubeadm: true diff --git a/filebeat/_meta/test/docs/01_playground/filebeat.yaml b/filebeat/_meta/test/docs/01_playground/filebeat.yaml index c16b68757e4..d4b4101c358 100644 --- a/filebeat/_meta/test/docs/01_playground/filebeat.yaml +++ b/filebeat/_meta/test/docs/01_playground/filebeat.yaml @@ -9,11 +9,20 @@ metadata: data: filebeat.yml: |- filebeat.inputs: - - type: container + - type: filestream + id: kubernetes-container-logs paths: - /var/log/containers/*.log + parsers: + - container: ~ + prospector: + scanner: + fingerprint.enabled: true + symlinks: true + file_identity.fingerprint: ~ processors: - add_kubernetes_metadata: + #use_kubeadm: true host: ${NODE_NAME} matchers: - logs_path: @@ -25,10 +34,19 @@ data: # - type: kubernetes # node: ${NODE_NAME} # hints.enabled: true + # #use_kubeadm: true # hints.default_config: - # type: container + # type: filestream + # id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id} # paths: - # - /var/log/containers/*${data.kubernetes.container.id}.log + # - /var/log/containers/*-${data.kubernetes.container.id}.log + # parsers: + # - container: ~ + # prospector: + # scanner: + # fingerprint.enabled: true + # symlinks: true + # file_identity.fingerprint: ~ processors: - add_cloud_metadata: @@ -64,12 +82,12 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: filebeat - image: docker.elastic.co/beats/filebeat:8.0.0-SNAPSHOT + image: docker.elastic.co/beats/filebeat:8.16.0-SNAPSHOT command: [ "sleep" ] args: [ "infinity" ] env: - name: ELASTICSEARCH_HOST - value: elasticsearch.default.svc.cluster.local + value: elasticsearch - name: ELASTICSEARCH_PORT value: "9200" - name: ELASTICSEARCH_USERNAME diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 55b0290e21c..14e9f276fb4 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1219,6 +1219,30 @@ filebeat.inputs: # paths: # - /var/log/containers/*.log +#Example: for kubernetes container logs autodiscovery +# filebeat.autodiscover: +# providers: +# - type: kubernetes +# node: ${NODE_NAME} +# hints.enabled: true +# # By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. +# use_kubeadm: true +# hints.default_config: +# type: filestream +# id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id} +# paths: +# - /var/log/containers/*-${data.kubernetes.container.id}.log +# parsers: +# - container: ~ +# prospector: +# scanner: +# fingerprint.enabled: true +# symlinks: true +# file_identity.fingerprint: ~ + +#By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. +# use_kubeadm: true + # ========================== Filebeat global options =========================== # Registry data path. If a relative path is used, it is considered relative to the diff --git a/libbeat/autodiscover/providers/kubernetes/config.go b/libbeat/autodiscover/providers/kubernetes/config.go index b1ec09900fd..36dc117db50 100644 --- a/libbeat/autodiscover/providers/kubernetes/config.go +++ b/libbeat/autodiscover/providers/kubernetes/config.go @@ -37,6 +37,7 @@ var AllSupportedHints = []string{"enabled", "module", "metricsets", "hosts", "pe type Config struct { KubeConfig string `config:"kube_config"` KubeClientOptions kubernetes.KubeClientOptions `config:"kube_client_options"` + KubeAdm bool `config:"use_kubeadm"` Namespace string `config:"namespace"` SyncPeriod time.Duration `config:"sync_period"` @@ -71,6 +72,7 @@ var DefaultCleanupTimeout time.Duration = 0 func defaultConfig() *Config { return &Config{ SyncPeriod: 10 * time.Minute, + KubeAdm: true, Resource: "pod", CleanupTimeout: DefaultCleanupTimeout, Prefix: "co.elastic", diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index 764dab44f46..be7179873ec 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -97,6 +97,15 @@ func NewPodEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publish fu } metaConf := config.AddResourceMetadata + // We initialise the use_kubeadm variable based on modules KubeAdm base configuration + err = metaConf.Namespace.SetBool("use_kubeadm", -1, config.KubeAdm) + if err != nil { + logger.Errorf("couldn't set kubeadm variable for namespace due to error %+v", err) + } + err = metaConf.Node.SetBool("use_kubeadm", -1, config.KubeAdm) + if err != nil { + logger.Errorf("couldn't set kubeadm variable for node due to error %+v", err) + } if metaConf.Node.Enabled() || config.Hints.Enabled() { options := kubernetes.WatchOptions{ diff --git a/libbeat/autodiscover/providers/kubernetes/service.go b/libbeat/autodiscover/providers/kubernetes/service.go index c1604cd0fe4..59b4f15f1ee 100644 --- a/libbeat/autodiscover/providers/kubernetes/service.go +++ b/libbeat/autodiscover/providers/kubernetes/service.go @@ -71,6 +71,11 @@ func NewServiceEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publis var namespaceWatcher kubernetes.Watcher metaConf := config.AddResourceMetadata + // We initialise the use_kubeadm variable based on modules KubeAdm base configuration + err = metaConf.Namespace.SetBool("use_kubeadm", -1, config.KubeAdm) + if err != nil { + logger.Errorf("couldn't set kubeadm variable for namespace due to error %+v", err) + } if metaConf.Namespace.Enabled() || config.Hints.Enabled() { namespaceWatcher, err = kubernetes.NewNamedWatcher("namespace", client, &kubernetes.Namespace{}, kubernetes.WatchOptions{ diff --git a/libbeat/processors/add_kubernetes_metadata/config.go b/libbeat/processors/add_kubernetes_metadata/config.go index 7c74c82268d..aaf219f8642 100644 --- a/libbeat/processors/add_kubernetes_metadata/config.go +++ b/libbeat/processors/add_kubernetes_metadata/config.go @@ -29,6 +29,7 @@ import ( type kubeAnnotatorConfig struct { KubeConfig string `config:"kube_config"` KubeClientOptions kubernetes.KubeClientOptions `config:"kube_client_options"` + KubeAdm bool `config:"use_kubeadm"` Node string `config:"node"` Scope string `config:"scope"` Namespace string `config:"namespace"` diff --git a/libbeat/processors/add_kubernetes_metadata/docs/add_kubernetes_metadata.asciidoc b/libbeat/processors/add_kubernetes_metadata/docs/add_kubernetes_metadata.asciidoc index 06cfcbf5340..df795135803 100644 --- a/libbeat/processors/add_kubernetes_metadata/docs/add_kubernetes_metadata.asciidoc +++ b/libbeat/processors/add_kubernetes_metadata/docs/add_kubernetes_metadata.asciidoc @@ -152,6 +152,7 @@ Example: ------------------------------------------------------------------------------------- `kube_config`:: (Optional) Use given config file as configuration for Kubernetes client. It defaults to `KUBECONFIG` environment variable if present. +`use_kubeadm`:: (Optional) Default true. By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. `kube_client_options`:: (Optional) Additional options can be configured for Kubernetes client. Currently client QPS and burst are supported, if not set Kubernetes client's https://pkg.go.dev/k8s.io/client-go/rest#pkg-constants[default QPS and burst] will be used. diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index 2dedf99d236..c22875ccf3c 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -147,6 +147,15 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *config.C) { k.initOnce.Do(func() { var replicaSetWatcher, jobWatcher, namespaceWatcher, nodeWatcher kubernetes.Watcher + // We initialise the use_kubeadm variable based on modules KubeAdm base configuration + err := config.AddResourceMetadata.Namespace.SetBool("use_kubeadm", -1, config.KubeAdm) + if err != nil { + k.log.Errorf("couldn't set kubeadm variable for namespace due to error %+v", err) + } + err = config.AddResourceMetadata.Node.SetBool("use_kubeadm", -1, config.KubeAdm) + if err != nil { + k.log.Errorf("couldn't set kubeadm variable for node due to error %+v", err) + } client, err := kubernetes.GetKubernetesClient(config.KubeConfig, config.KubeClientOptions) if err != nil { if kubernetes.IsInCluster(config.KubeConfig) { diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index 53b6233764f..4d030e05d72 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -235,6 +235,8 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true #include_labels: [] #exclude_labels: [] #include_annotations: [] @@ -286,6 +288,8 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true #include_labels: [] #exclude_labels: [] #include_annotations: [] @@ -326,6 +330,8 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Set the namespace to watch for events #namespace: staging # Set the sync period of the watchers @@ -346,6 +352,8 @@ metricbeat.modules: ssl.certificate_authorities: - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt period: 30s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Kubernetes proxy server # (when running metricbeat locally at hosts or as a daemonset + host network) @@ -355,6 +363,8 @@ metricbeat.modules: - proxy hosts: ["localhost:10249"] period: 10s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Kubernetes controller manager # (URL and deployment method should be adapted to match the controller manager deployment / service / endpoint) @@ -364,6 +374,8 @@ metricbeat.modules: - controllermanager hosts: ["http://localhost:10252"] period: 10s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Kubernetes scheduler # (URL and deployment method should be adapted to match scheduler deployment / service / endpoint) @@ -373,6 +385,8 @@ metricbeat.modules: - scheduler hosts: ["localhost:10251"] period: 10s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true ---- This module supports TLS connections when using `ssl` config field, as described in <>. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 288ad96d29c..c9de9366a74 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -522,6 +522,8 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true #include_labels: [] #exclude_labels: [] #include_annotations: [] @@ -573,6 +575,8 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true #include_labels: [] #exclude_labels: [] #include_annotations: [] @@ -613,6 +617,8 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Set the namespace to watch for events #namespace: staging # Set the sync period of the watchers @@ -633,6 +639,8 @@ metricbeat.modules: ssl.certificate_authorities: - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt period: 30s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Kubernetes proxy server # (when running metricbeat locally at hosts or as a daemonset + host network) @@ -642,6 +650,8 @@ metricbeat.modules: - proxy hosts: ["localhost:10249"] period: 10s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Kubernetes controller manager # (URL and deployment method should be adapted to match the controller manager deployment / service / endpoint) @@ -651,6 +661,8 @@ metricbeat.modules: - controllermanager hosts: ["http://localhost:10252"] period: 10s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Kubernetes scheduler # (URL and deployment method should be adapted to match scheduler deployment / service / endpoint) @@ -660,6 +672,8 @@ metricbeat.modules: - scheduler hosts: ["localhost:10251"] period: 10s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true #--------------------------------- KVM Module --------------------------------- - module: kvm diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml index 23f5ce8dea6..2cf519bc320 100644 --- a/metricbeat/module/kubernetes/_meta/config.reference.yml +++ b/metricbeat/module/kubernetes/_meta/config.reference.yml @@ -21,6 +21,8 @@ # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true #include_labels: [] #exclude_labels: [] #include_annotations: [] @@ -72,6 +74,8 @@ # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true #include_labels: [] #exclude_labels: [] #include_annotations: [] @@ -112,6 +116,8 @@ # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Set the namespace to watch for events #namespace: staging # Set the sync period of the watchers @@ -132,6 +138,8 @@ ssl.certificate_authorities: - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt period: 30s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Kubernetes proxy server # (when running metricbeat locally at hosts or as a daemonset + host network) @@ -141,6 +149,8 @@ - proxy hosts: ["localhost:10249"] period: 10s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Kubernetes controller manager # (URL and deployment method should be adapted to match the controller manager deployment / service / endpoint) @@ -150,6 +160,8 @@ - controllermanager hosts: ["http://localhost:10252"] period: 10s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Kubernetes scheduler # (URL and deployment method should be adapted to match scheduler deployment / service / endpoint) @@ -159,3 +171,5 @@ - scheduler hosts: ["localhost:10251"] period: 10s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index f94d424ec0e..d0f041edfe7 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -592,6 +592,18 @@ func NewResourceMetadataEnricher( var specificMetaGen metadata.MetaGen var generalMetaGen *metadata.Resource +<<<<<<< HEAD +======= + // We initialise the use_kubeadm variable based on modules KubeAdm base configuration + err = config.AddResourceMetadata.Namespace.SetBool("use_kubeadm", -1, commonMetaConfig.KubeAdm) + if err != nil { + log.Errorf("couldn't set kubeadm variable for namespace due to error %+v", err) + } + err = config.AddResourceMetadata.Node.SetBool("use_kubeadm", -1, commonMetaConfig.KubeAdm) + if err != nil { + log.Errorf("couldn't set kubeadm variable for node due to error %+v", err) + } +>>>>>>> 89cd840997 (Disable kubeadm for filebeat (both filbeat.input and filbeat.autodiscover) (#40301)) // Create the metadata generator to be used in the watcher's event handler. // Both specificMetaGen and generalMetaGen implement Generate method for metadata collection. if resourceName == ServiceResource || resourceName == PodResource { @@ -755,6 +767,18 @@ func NewContainerMetadataEnricher( log.Errorf("Error starting the watchers: %s", err) return &nilEnricher{} } +<<<<<<< HEAD +======= + // We initialise the use_kubeadm variable based on modules KubeAdm base configuration + err = config.AddResourceMetadata.Namespace.SetBool("use_kubeadm", -1, commonMetaConfig.KubeAdm) + if err != nil { + log.Errorf("couldn't set kubeadm variable for namespace due to error %+v", err) + } + err = config.AddResourceMetadata.Node.SetBool("use_kubeadm", -1, commonMetaConfig.KubeAdm) + if err != nil { + log.Errorf("couldn't set kubeadm variable for node due to error %+v", err) + } +>>>>>>> 89cd840997 (Disable kubeadm for filebeat (both filbeat.input and filbeat.autodiscover) (#40301)) metaGen, err := createMetadataGenSpecific(client, commonConfig, config.AddResourceMetadata, PodResource, resourceWatchers) if err != nil { diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index d867bdad8af..09a540aa21e 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3146,6 +3146,30 @@ filebeat.inputs: # paths: # - /var/log/containers/*.log +#Example: for kubernetes container logs autodiscovery +# filebeat.autodiscover: +# providers: +# - type: kubernetes +# node: ${NODE_NAME} +# hints.enabled: true +# # By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. +# use_kubeadm: true +# hints.default_config: +# type: filestream +# id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id} +# paths: +# - /var/log/containers/*-${data.kubernetes.container.id}.log +# parsers: +# - container: ~ +# prospector: +# scanner: +# fingerprint.enabled: true +# symlinks: true +# file_identity.fingerprint: ~ + +#By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. +# use_kubeadm: true + # ========================== Filebeat global options =========================== # Registry data path. If a relative path is used, it is considered relative to the diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 94670887e76..c1821be998b 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -921,6 +921,8 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true #include_labels: [] #exclude_labels: [] #include_annotations: [] @@ -972,6 +974,8 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true #include_labels: [] #exclude_labels: [] #include_annotations: [] @@ -1012,6 +1016,8 @@ metricbeat.modules: # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Set the namespace to watch for events #namespace: staging # Set the sync period of the watchers @@ -1032,6 +1038,8 @@ metricbeat.modules: ssl.certificate_authorities: - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt period: 30s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Kubernetes proxy server # (when running metricbeat locally at hosts or as a daemonset + host network) @@ -1041,6 +1049,8 @@ metricbeat.modules: - proxy hosts: ["localhost:10249"] period: 10s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Kubernetes controller manager # (URL and deployment method should be adapted to match the controller manager deployment / service / endpoint) @@ -1050,6 +1060,8 @@ metricbeat.modules: - controllermanager hosts: ["http://localhost:10252"] period: 10s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true # Kubernetes scheduler # (URL and deployment method should be adapted to match scheduler deployment / service / endpoint) @@ -1059,6 +1071,8 @@ metricbeat.modules: - scheduler hosts: ["localhost:10251"] period: 10s + #By default requests to kubeadm config map are made in order to enrich cluster name by requesting /api/v1/namespaces/kube-system/configmaps/kubeadm-config API endpoint. + use_kubeadm: true #--------------------------------- KVM Module --------------------------------- - module: kvm