From 101cf07601fffbae74fd554a9ada40ecb172626c Mon Sep 17 00:00:00 2001 From: RagalahariP Date: Thu, 18 May 2023 12:39:15 +0530 Subject: [PATCH 1/6] Updating helm charts to support version 3 windows integration --- charts/newrelic-infrastructure/README.md | 3 + .../templates/NOTES.txt | 4 -- .../templates/_helpers_compatibility.tpl | 7 -- .../templates/clusterrole.yaml | 1 + .../templates/kubelet/_naming.tpl | 4 ++ .../templates/kubelet/daemonset-windows.yaml | 70 +++++++++++++++++++ .../templates/kubelet/daemonset.yaml | 3 +- .../tests/nodeSelectors_test.yaml | 6 ++ charts/newrelic-infrastructure/values.yaml | 13 ++++ 9 files changed, 99 insertions(+), 12 deletions(-) create mode 100644 charts/newrelic-infrastructure/templates/kubelet/daemonset-windows.yaml diff --git a/charts/newrelic-infrastructure/README.md b/charts/newrelic-infrastructure/README.md index 56cadee19..7decb9ba9 100644 --- a/charts/newrelic-infrastructure/README.md +++ b/charts/newrelic-infrastructure/README.md @@ -161,6 +161,7 @@ integrations that you have configured. | kubelet.config.retries | int | `3` | Number of retries after timeout expired | | kubelet.config.scraperMaxReruns | int | `4` | Max number of scraper rerun when scraper runtime error happens | | kubelet.config.timeout | string | `"10s"` | Timeout for the kubelet APIs contacted by the integration | +| kubelet.enableWindows | bool | `false` | Enable Windows monitoring and update `windowsOsList` with the list of Windows versions present in the cluster. | | kubelet.enabled | bool | `true` | Enable kubelet monitoring. Advanced users only. Setting this to `false` is not supported and will break the New Relic experience. | | kubelet.extraEnv | list | `[]` | Add user environment variables to the agent | | kubelet.extraEnvFrom | list | `[]` | Add user environment from configMaps or secrets as variables to the agent | @@ -168,6 +169,8 @@ integrations that you have configured. | kubelet.extraVolumes | list | `[]` | Volumes to mount in the containers | | kubelet.hostNetwork | bool | Not set | Sets pod's hostNetwork. When set bypasses global/common variable | | kubelet.tolerations | list | Schedules in all tainted nodes | Tolerations for the control plane DaemonSet. | +| kubelet.enableWindows | bool | `false` | Set to enable Windows node monitoring. | +| kubelet.windowsOsList | list | `[]` | List of Windows versions present in the cluster. Our Kubernetes integration supported Windows versions LTSC 2019 (1809), 20H2, and LTSC 2022 | | labels | object | `{}` | Additional labels for chart objects. Can be configured also with `global.labels` | | licenseKey | string | `""` | This set this license key to use. Can be configured also with `global.licenseKey` | | lowDataMode | bool | `false` (See [Low data mode](README.md#low-data-mode)) | Send less data by incrementing the interval from `15s` (the default when `lowDataMode` is `false` or `nil`) to `30s`. Non-nil values of `common.config.interval` will override this value. | diff --git a/charts/newrelic-infrastructure/templates/NOTES.txt b/charts/newrelic-infrastructure/templates/NOTES.txt index 16cc6ea13..734174c5a 100644 --- a/charts/newrelic-infrastructure/templates/NOTES.txt +++ b/charts/newrelic-infrastructure/templates/NOTES.txt @@ -110,10 +110,6 @@ future. Please migrate your agent config to the new format in the `common.agentC {{ $errors = printf "%s\n\n%s" $errors (include "newrelic.compatibility.message.image" . ) }} {{- end }} -{{- if .Values.enableWindows }} -{{ $errors = printf "%s\n\n%s" $errors (include "newrelic.compatibility.message.windows" . ) }} -{{- end }} - {{- if ( or .Values.controllerManagerEndpointUrl .Values.schedulerEndpointUrl .Values.etcdEndpointUrl .Values.apiServerEndpointUrl )}} {{ $errors = printf "%s\n\n%s" $errors (include "newrelic.compatibility.message.apiURL" . ) }} {{- end }} diff --git a/charts/newrelic-infrastructure/templates/_helpers_compatibility.tpl b/charts/newrelic-infrastructure/templates/_helpers_compatibility.tpl index 07365e5a1..093254510 100644 --- a/charts/newrelic-infrastructure/templates/_helpers_compatibility.tpl +++ b/charts/newrelic-infrastructure/templates/_helpers_compatibility.tpl @@ -123,13 +123,6 @@ Please configure the API Server port as a part of 'apiServer.autodiscover[].endp ------ {{- end -}} -{{- define "newrelic.compatibility.message.windows" -}} -nri-kubernetes v3 does not support deploying into windows Nodes. -Please use the latest 2.x version of the chart. - ------- -{{- end -}} - {{- define "newrelic.compatibility.message.etcdSecrets" -}} Values "etcdTlsSecretName" and "etcdTlsSecretNamespace" are no longer supported, please specify them as a part of the 'etcd' config in the values, for example: diff --git a/charts/newrelic-infrastructure/templates/clusterrole.yaml b/charts/newrelic-infrastructure/templates/clusterrole.yaml index 4913448e7..79e0dd4a7 100644 --- a/charts/newrelic-infrastructure/templates/clusterrole.yaml +++ b/charts/newrelic-infrastructure/templates/clusterrole.yaml @@ -18,6 +18,7 @@ rules: - "services" - "nodes" - "namespaces" + - "pods" # For Windows integration, Pods required in list of resources verbs: [ "get", "list", "watch" ] - nonResourceURLs: ["/metrics"] verbs: ["get"] diff --git a/charts/newrelic-infrastructure/templates/kubelet/_naming.tpl b/charts/newrelic-infrastructure/templates/kubelet/_naming.tpl index 71c142156..397277c39 100644 --- a/charts/newrelic-infrastructure/templates/kubelet/_naming.tpl +++ b/charts/newrelic-infrastructure/templates/kubelet/_naming.tpl @@ -3,6 +3,10 @@ {{- include "newrelic.common.naming.truncateToDNSWithSuffix" (dict "name" (include "nriKubernetes.naming.fullname" .) "suffix" "kubelet") -}} {{- end -}} +{{- define "nriKubernetes.kubelet.windows.fullname" -}} +{{- include "newrelic.common.naming.truncateToDNSWithSuffix" (dict "name" (include "nriKubernetes.naming.fullname" .) "suffix" "windows") -}} +{{- end -}} + {{- define "nriKubernetes.kubelet.fullname.agent" -}} {{- include "newrelic.common.naming.truncateToDNSWithSuffix" (dict "name" (include "nriKubernetes.naming.fullname" .) "suffix" "agent-kubelet") -}} {{- end -}} diff --git a/charts/newrelic-infrastructure/templates/kubelet/daemonset-windows.yaml b/charts/newrelic-infrastructure/templates/kubelet/daemonset-windows.yaml new file mode 100644 index 000000000..8f4e016a0 --- /dev/null +++ b/charts/newrelic-infrastructure/templates/kubelet/daemonset-windows.yaml @@ -0,0 +1,70 @@ +{{- if and .Values.kubelet.enabled .Values.kubelet.enableWindows }} +{{- range .Values.kubelet.windowsOsList }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + namespace: {{ $.Release.Namespace }} + name: {{ include "nriKubernetes.kubelet.windows.fullname" $ }}-{{ .version }} + {{- $legacyAnnotation:= fromYaml (include "newrelic.compatibility.annotations" $) -}} + {{- with include "newrelic.compatibility.valueWithFallback" (dict "legacy" $legacyAnnotation "supported" $.Values.kubelet.annotations )}} + annotations: {{ . | nindent 4 }} + {{- end }} +spec: + {{- with $.Values.updateStrategy }} + updateStrategy: {{ toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "newrelic.common.labels.selectorLabels" $ | nindent 6 }} + app.kubernetes.io/component: kubelet + template: + metadata: + annotations: + checksum/nri-kubernetes: {{ include (print $.Template.BasePath "/kubelet/scraper-configmap.yaml") $ | sha256sum }} + checksum/agent-config: {{ include (print $.Template.BasePath "/kubelet/agent-configmap.yaml") $ | sha256sum }} + {{- if include "newrelic.common.license.secret" $ }}{{- /* If the is secret to template */}} + checksum/license-secret: {{ include (print $.Template.BasePath "/secret.yaml") $ | sha256sum }} + {{- end }} + labels: + {{- include "nriKubernetes.labels.podLabels" $ | nindent 8 }} + app.kubernetes.io/component: kubelet + spec: + {{- with include "newrelic.common.dnsConfig" $ }} + dnsConfig: + {{- . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "newrelic.common.serviceAccount.name" $ }} + containers: + - name: agent + image: newrelic/infrastructure-k8s:{{ .imageTag }} + imagePullPolicy: {{ $.Values.images.integration.pullPolicy }} + env: + - name: NRIA_LICENSE_KEY + valueFrom: + secretKeyRef: + name: {{ include "newrelic.common.license.secretName" $ }} + key: {{ include "newrelic.common.license.secretKeyName" $ }} + - name: "NRI_KUBERNETES_CLUSTERNAME" + value: {{ include "newrelic.common.cluster" $ }} + - name: "NRI_KUBERNETES_VERBOSE" + value: {{ include "newrelic.common.verboseLog.valueAsBoolean" $ | quote }} + - name: "NRK8S_NODE_NAME" + valueFrom: + fieldRef: + apiVersion: "v1" + fieldPath: "spec.nodeName" + - name: "CLUSTER_NAME" + value: {{ include "newrelic.common.cluster" $ }} + - name: "NRIA_PASSTHROUGH_ENVIRONMENT" + value: "KUBERNETES_SERVICE_HOST,KUBERNETES_SERVICE_PORT,CLUSTER_NAME,CADVISOR_PORT,NRK8S_NODE_NAME,KUBE_STATE_METRICS_URL,ETCD_TLS_SECRET_NAME,ETCD_TLS_SECRET_NAMESPACE,API_SERVER_SECURE_PORT,NRIA_CACHE_PATH,TIMEOUT,DISCOVERY_CACHE_TTL" + nodeSelector: + node.kubernetes.io/windows-build: {{ .buildNumber }} + kubernetes.io/os: windows + tolerations: + - key: "node.kubernetes.io/os" + operator: "Equal" + value: "windows" + effect: "NoSchedule" +--- +{{- end }} +{{- end }} diff --git a/charts/newrelic-infrastructure/templates/kubelet/daemonset.yaml b/charts/newrelic-infrastructure/templates/kubelet/daemonset.yaml index a725a3a13..5fb2eb724 100644 --- a/charts/newrelic-infrastructure/templates/kubelet/daemonset.yaml +++ b/charts/newrelic-infrastructure/templates/kubelet/daemonset.yaml @@ -251,8 +251,9 @@ spec: tolerations: {{- . | nindent 8 }} {{- end }} - {{- with .Values.kubelet.nodeSelector | default (fromYaml (include "newrelic.common.nodeSelector" .)) }} nodeSelector: + kubernetes.io/os: linux + {{- with .Values.kubelet.nodeSelector | default (fromYaml (include "newrelic.common.nodeSelector" .)) }} {{- toYaml . | nindent 8 }} {{- end -}} {{- end }} diff --git a/charts/newrelic-infrastructure/tests/nodeSelectors_test.yaml b/charts/newrelic-infrastructure/tests/nodeSelectors_test.yaml index 7644a951c..920de3a17 100644 --- a/charts/newrelic-infrastructure/tests/nodeSelectors_test.yaml +++ b/charts/newrelic-infrastructure/tests/nodeSelectors_test.yaml @@ -30,6 +30,7 @@ tests: path: spec.template.spec.nodeSelector value: disktype: ssd + kubernetes.io/os: linux template: templates/kubelet/daemonset.yaml - equal: path: spec.template.spec.nodeSelector @@ -53,6 +54,7 @@ tests: path: spec.template.spec.nodeSelector value: disktype: ssd + kubernetes.io/os: linux template: templates/kubelet/daemonset.yaml - equal: path: spec.template.spec.nodeSelector @@ -79,6 +81,7 @@ tests: path: spec.template.spec.nodeSelector value: disktype: real + kubernetes.io/os: linux template: templates/kubelet/daemonset.yaml - equal: path: spec.template.spec.nodeSelector @@ -105,6 +108,7 @@ tests: path: spec.template.spec.nodeSelector value: disktype: ssd + kubernetes.io/os: linux template: templates/kubelet/daemonset.yaml - equal: path: spec.template.spec.nodeSelector @@ -131,6 +135,7 @@ tests: path: spec.template.spec.nodeSelector value: disktype: real + kubernetes.io/os: linux template: templates/kubelet/daemonset.yaml - equal: path: spec.template.spec.nodeSelector @@ -157,6 +162,7 @@ tests: path: spec.template.spec.nodeSelector value: disktype: ssd + kubernetes.io/os: linux template: templates/kubelet/daemonset.yaml - equal: path: spec.template.spec.nodeSelector diff --git a/charts/newrelic-infrastructure/values.yaml b/charts/newrelic-infrastructure/values.yaml index f62105f99..f10cc629c 100644 --- a/charts/newrelic-infrastructure/values.yaml +++ b/charts/newrelic-infrastructure/values.yaml @@ -83,6 +83,19 @@ kubelet: # -- Enable kubelet monitoring. # Advanced users only. Setting this to `false` is not supported and will break the New Relic experience. enabled: true + # -- Enable Windows monitoring and update `windowsOsList` with the list of Windows versions present in the cluster. + enableWindows: false + # -- List of Windows versions present in the cluster. Our Kubernetes integration supported Windows versions LTSC 2019 (1809), 20H2, and LTSC 2022 + windowsOsList: + # - version: 2019 + # imageTag: 2-windows-1809-alpha + # buildNumber: 10.0.17763 + # - version: 2022 + # imageTag: 2-windows-ltsc2022-alpha + # buildNumber: 10.0.20348 + # - version: 20h2 + # imageTag: 2-windows-20H2-alpha + # buildNumber: 10.0.19042 annotations: {} # -- Tolerations for the control plane DaemonSet. # @default -- Schedules in all tainted nodes From ba8abf705621f17fbdcac8e64ad496591f29fdd8 Mon Sep 17 00:00:00 2001 From: RagalahariP Date: Thu, 18 May 2023 12:48:25 +0530 Subject: [PATCH 2/6] Updating README with helm-docs --- charts/newrelic-infrastructure/README.md | 1 - charts/newrelic-infrastructure/values.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/newrelic-infrastructure/README.md b/charts/newrelic-infrastructure/README.md index 7decb9ba9..07c6a78e3 100644 --- a/charts/newrelic-infrastructure/README.md +++ b/charts/newrelic-infrastructure/README.md @@ -169,7 +169,6 @@ integrations that you have configured. | kubelet.extraVolumes | list | `[]` | Volumes to mount in the containers | | kubelet.hostNetwork | bool | Not set | Sets pod's hostNetwork. When set bypasses global/common variable | | kubelet.tolerations | list | Schedules in all tainted nodes | Tolerations for the control plane DaemonSet. | -| kubelet.enableWindows | bool | `false` | Set to enable Windows node monitoring. | | kubelet.windowsOsList | list | `[]` | List of Windows versions present in the cluster. Our Kubernetes integration supported Windows versions LTSC 2019 (1809), 20H2, and LTSC 2022 | | labels | object | `{}` | Additional labels for chart objects. Can be configured also with `global.labels` | | licenseKey | string | `""` | This set this license key to use. Can be configured also with `global.licenseKey` | diff --git a/charts/newrelic-infrastructure/values.yaml b/charts/newrelic-infrastructure/values.yaml index f10cc629c..d3385ce91 100644 --- a/charts/newrelic-infrastructure/values.yaml +++ b/charts/newrelic-infrastructure/values.yaml @@ -86,7 +86,7 @@ kubelet: # -- Enable Windows monitoring and update `windowsOsList` with the list of Windows versions present in the cluster. enableWindows: false # -- List of Windows versions present in the cluster. Our Kubernetes integration supported Windows versions LTSC 2019 (1809), 20H2, and LTSC 2022 - windowsOsList: + windowsOsList: [] # - version: 2019 # imageTag: 2-windows-1809-alpha # buildNumber: 10.0.17763 From 0230975fde2a28b873d9919b01883310526a611d Mon Sep 17 00:00:00 2001 From: RagalahariP Date: Mon, 22 May 2023 23:25:46 +0530 Subject: [PATCH 3/6] Updated comments --- charts/newrelic-infrastructure/README.md | 2 +- charts/newrelic-infrastructure/values.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/newrelic-infrastructure/README.md b/charts/newrelic-infrastructure/README.md index 07c6a78e3..675b3a6cd 100644 --- a/charts/newrelic-infrastructure/README.md +++ b/charts/newrelic-infrastructure/README.md @@ -169,7 +169,7 @@ integrations that you have configured. | kubelet.extraVolumes | list | `[]` | Volumes to mount in the containers | | kubelet.hostNetwork | bool | Not set | Sets pod's hostNetwork. When set bypasses global/common variable | | kubelet.tolerations | list | Schedules in all tainted nodes | Tolerations for the control plane DaemonSet. | -| kubelet.windowsOsList | list | `[]` | List of Windows versions present in the cluster. Our Kubernetes integration supported Windows versions LTSC 2019 (1809), 20H2, and LTSC 2022 | +| kubelet.windowsOsList | list | `[]` | `windowsOsList` is only required if `enableWindows` is true. It contains list of Windows versions present in the cluster. Our Kubernetes integration supported Windows versions LTSC 2019 (1809), 20H2, and LTSC 2022 | | labels | object | `{}` | Additional labels for chart objects. Can be configured also with `global.labels` | | licenseKey | string | `""` | This set this license key to use. Can be configured also with `global.licenseKey` | | lowDataMode | bool | `false` (See [Low data mode](README.md#low-data-mode)) | Send less data by incrementing the interval from `15s` (the default when `lowDataMode` is `false` or `nil`) to `30s`. Non-nil values of `common.config.interval` will override this value. | diff --git a/charts/newrelic-infrastructure/values.yaml b/charts/newrelic-infrastructure/values.yaml index d3385ce91..ddf940de0 100644 --- a/charts/newrelic-infrastructure/values.yaml +++ b/charts/newrelic-infrastructure/values.yaml @@ -85,7 +85,8 @@ kubelet: enabled: true # -- Enable Windows monitoring and update `windowsOsList` with the list of Windows versions present in the cluster. enableWindows: false - # -- List of Windows versions present in the cluster. Our Kubernetes integration supported Windows versions LTSC 2019 (1809), 20H2, and LTSC 2022 + # -- `windowsOsList` is only required if `enableWindows` is true. It contains list of Windows versions present in the cluster. + # Our Kubernetes integration supported Windows versions LTSC 2019 (1809), 20H2, and LTSC 2022 windowsOsList: [] # - version: 2019 # imageTag: 2-windows-1809-alpha From ec2094a2103d65baca6f535895db9069b7e32b84 Mon Sep 17 00:00:00 2001 From: RagalahariP Date: Thu, 25 May 2023 17:00:24 +0530 Subject: [PATCH 4/6] Adding node selector for KSM so that it will ensure to run for only Linux nodes. And collecting metrics from Windows nodes will continue the same approach in V2 --- charts/newrelic-infrastructure/templates/ksm/deployment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/newrelic-infrastructure/templates/ksm/deployment.yaml b/charts/newrelic-infrastructure/templates/ksm/deployment.yaml index c036ba653..a923a77f6 100644 --- a/charts/newrelic-infrastructure/templates/ksm/deployment.yaml +++ b/charts/newrelic-infrastructure/templates/ksm/deployment.yaml @@ -184,8 +184,9 @@ spec: tolerations: {{- . | nindent 8 }} {{- end }} - {{- with .Values.ksm.nodeSelector | default (fromYaml (include "newrelic.common.nodeSelector" .)) }} nodeSelector: + kubernetes.io/os: linux + {{- with .Values.ksm.nodeSelector | default (fromYaml (include "newrelic.common.nodeSelector" .)) }} {{- toYaml . | nindent 8 }} {{- end -}} {{- end }} From a399c401186246cdb089fbc5240546f15779bc1c Mon Sep 17 00:00:00 2001 From: RagalahariP Date: Thu, 25 May 2023 17:14:34 +0530 Subject: [PATCH 5/6] Fixing test cases --- .../newrelic-infrastructure/tests/nodeSelectors_test.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/charts/newrelic-infrastructure/tests/nodeSelectors_test.yaml b/charts/newrelic-infrastructure/tests/nodeSelectors_test.yaml index 920de3a17..01e17f61a 100644 --- a/charts/newrelic-infrastructure/tests/nodeSelectors_test.yaml +++ b/charts/newrelic-infrastructure/tests/nodeSelectors_test.yaml @@ -25,6 +25,7 @@ tests: path: spec.template.spec.nodeSelector value: disktype: ssd + kubernetes.io/os: linux template: templates/ksm/deployment.yaml - equal: path: spec.template.spec.nodeSelector @@ -49,6 +50,7 @@ tests: path: spec.template.spec.nodeSelector value: disktype: ssd + kubernetes.io/os: linux template: templates/ksm/deployment.yaml - equal: path: spec.template.spec.nodeSelector @@ -76,6 +78,7 @@ tests: path: spec.template.spec.nodeSelector value: disktype: real + kubernetes.io/os: linux template: templates/ksm/deployment.yaml - equal: path: spec.template.spec.nodeSelector @@ -103,6 +106,7 @@ tests: path: spec.template.spec.nodeSelector value: disktype: real + kubernetes.io/os: linux template: templates/ksm/deployment.yaml - equal: path: spec.template.spec.nodeSelector @@ -130,6 +134,7 @@ tests: path: spec.template.spec.nodeSelector value: disktype: ssd + kubernetes.io/os: linux template: templates/ksm/deployment.yaml - equal: path: spec.template.spec.nodeSelector @@ -157,6 +162,7 @@ tests: path: spec.template.spec.nodeSelector value: disktype: ssd + kubernetes.io/os: linux template: templates/ksm/deployment.yaml - equal: path: spec.template.spec.nodeSelector From 7d860af29ecf235d5f47bcc20bbf4caf37865d80 Mon Sep 17 00:00:00 2001 From: RagalahariP Date: Thu, 1 Jun 2023 21:59:20 +0530 Subject: [PATCH 6/6] Updating comment in clusterRole --- charts/newrelic-infrastructure/templates/clusterrole.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/newrelic-infrastructure/templates/clusterrole.yaml b/charts/newrelic-infrastructure/templates/clusterrole.yaml index 79e0dd4a7..7bcc323d4 100644 --- a/charts/newrelic-infrastructure/templates/clusterrole.yaml +++ b/charts/newrelic-infrastructure/templates/clusterrole.yaml @@ -18,7 +18,7 @@ rules: - "services" - "nodes" - "namespaces" - - "pods" # For Windows integration, Pods required in list of resources + - "pods" # Needed for our Windows integration to be able to access KSM on a Linux node. verbs: [ "get", "list", "watch" ] - nonResourceURLs: ["/metrics"] verbs: ["get"]