feat (in-cluster): [ingress-controller] update nginx to v1.11.3 (#102)

* upgrade old reference to 1.9.0 to 1.11.3 * snippet annotation no longer allowed in 1.11.3 by default * be explicit about enabling metrics since it is being scraped * ensure the image is the expected one * nginx no longer requires escalation priviledges * nginx requires being able to read the root filesystem * ensure nginx is not runnning as root * add seccomp profile type * inlcude the admission validation service port * Service account for admission now includes the token automount in true * update certgen image version * update sec context to be aligned with new nginx job * update certgen image version for patch container * update sec context to be aligned with new nginx job patch and resinstate fsgroup for create * update deployment docs * bug fix: remove sha256 from acr image imports * bug fix: indent nginx manifest file properly and remove sha256 image names * bug fix: allow-snippet-annotations set to true otherwise workload webui fails * bug fix: add location to rg from e2e validation
mspnp · Nov 20, 2024 · d4e3fc1 · d4e3fc1
1 parent e66254d
commit d4e3fc1
Show file tree

Hide file tree

Showing 3 changed files with 56 additions and 37 deletions.
diff --git a/cluster-manifests/ingress-nginx/deployment.yaml b/cluster-manifests/ingress-nginx/deployment.yaml
@@ -1,4 +1,4 @@
-# Source: https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.9.0/deploy/static/provider/cloud/deploy.yaml
+# Source: https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.11.3/deploy/static/provider/cloud/deploy.yaml
 ---
 apiVersion: v1
 automountServiceAccountToken: true
@@ -9,14 +9,14 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx
   namespace: ingress-nginx
 ---
 apiVersion: v1
 data:
   #https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/
-  # This is our Azure Application Gateway range. The ONLY traffic that should be hitting
+  # This is the Azure Application Gateway subnet address space. The ONLY traffic that should be hitting
   # the ingress controller directly is AAG.
   whitelist-source-range: 10.240.5.0/24
   ssl-redirect: "false"
@@ -28,7 +28,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx-controller
   namespace: ingress-nginx
 ---
@@ -39,7 +39,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx
 rules:
 - apiGroups:
@@ -120,7 +120,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -139,7 +139,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx
   namespace: ingress-nginx
 rules:
@@ -229,7 +229,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx
   namespace: ingress-nginx
 roleRef:
@@ -249,7 +249,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx-controller-admission
   namespace: ingress-nginx
 spec:
@@ -272,7 +272,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx-controller
   namespace: ingress-nginx
   annotations:
@@ -304,7 +304,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
     pci-scope: out-of-scope
   name: ingress-nginx-controller
   namespace: ingress-nginx
@@ -331,7 +331,7 @@ spec:
         app.kubernetes.io/instance: ingress-nginx
         app.kubernetes.io/name: ingress-nginx
         app.kubernetes.io/part-of: ingress-nginx
-        app.kubernetes.io/version: 1.9.0
+        app.kubernetes.io/version: 1.11.3
         pci-scope: out-of-scope
     spec:
       containers:
@@ -347,6 +347,7 @@ spec:
         - --validating-webhook-certificate=/usr/local/certificates/cert
         - --validating-webhook-key=/usr/local/certificates/key
         - --v=3
+        - --enable-metrics=true
         env:
         - name: POD_NAME
           valueFrom:
@@ -358,7 +359,7 @@ spec:
               fieldPath: metadata.namespace
         - name: LD_PRELOAD
           value: /usr/local/lib/libmimalloc.so
-        image: YOUR_ACR.azurecr.io/live/ingress-nginx/controller:v1.9.0
+        image: YOUR_ACR.azurecr.io/live/ingress-nginx/controller:v1.11.3
         imagePullPolicy: IfNotPresent
         lifecycle:
           preStop:
@@ -382,7 +383,7 @@ spec:
           protocol: TCP
         - containerPort: 8443
           name: webhook
-          protocol: TCP          
+          protocol: TCP
         - containerPort: 10254
           name: metrics
           protocol: TCP
@@ -395,7 +396,7 @@ spec:
           initialDelaySeconds: 10
           periodSeconds: 10
           successThreshold: 1
-          timeoutSeconds: 1        
+          timeoutSeconds: 1
         resources:
           requests:
             cpu: 100m
@@ -404,13 +405,17 @@ spec:
             cpu: 500m
             memory: 300Mi
         securityContext:
-          allowPrivilegeEscalation: true
+          allowPrivilegeEscalation: false
           capabilities:
             add:
             - NET_BIND_SERVICE
             drop:
             - ALL
           runAsUser: 101
+          readOnlyRootFilesystem: false
+          runAsNonRoot: true
+          seccompProfile:
+            type: RuntimeDefault
         volumeMounts:
           - name: webhook-cert
             mountPath: /usr/local/certificates/
@@ -443,7 +448,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: nginx
 spec:
   controller: k8s.io/ingress-nginx
@@ -456,7 +461,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx-admission
 webhooks:
 - admissionReviewVersions:
@@ -466,6 +471,7 @@ webhooks:
       name: ingress-nginx-controller-admission
       namespace: ingress-nginx
       path: /networking/v1/ingresses
+      port: 443
   failurePolicy: Fail
   matchPolicy: Equivalent
   name: validate.nginx.ingress.kubernetes.io
@@ -482,14 +488,15 @@ webhooks:
   sideEffects: None
 ---
 apiVersion: v1
+automountServiceAccountToken: true
 kind: ServiceAccount
 metadata:
   labels:
     app.kubernetes.io/component: admission-webhook
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx-admission
   namespace: ingress-nginx
 ---
@@ -501,7 +508,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx-admission
 rules:
 - apiGroups:
@@ -520,7 +527,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx-admission
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -539,7 +546,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx-admission
   namespace: ingress-nginx
 rules:
@@ -559,7 +566,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
   name: ingress-nginx-admission
   namespace: ingress-nginx
 roleRef:
@@ -579,7 +586,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
     pci-scope: out-of-scope
   name: ingress-nginx-admission-create
   namespace: ingress-nginx
@@ -592,7 +599,7 @@ spec:
         app.kubernetes.io/instance: ingress-nginx
         app.kubernetes.io/name: ingress-nginx
         app.kubernetes.io/part-of: ingress-nginx
-        app.kubernetes.io/version: 1.9.0
+        app.kubernetes.io/version: 1.11.3
         pci-scope: out-of-scope
     spec:
       containers:
@@ -606,11 +613,17 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: YOUR_ACR.azurecr.io/live/ingress-nginx/kube-webhook-certgen:v20230407
+        image: YOUR_ACR.azurecr.io/live/ingress-nginx/kube-webhook-certgen:v1.4.4
         imagePullPolicy: IfNotPresent
         name: create
         securityContext:
           allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          seccompProfile:
+            type: RuntimeDefault
         resources:
           requests:
             cpu: 100m
@@ -623,9 +636,9 @@ spec:
         pci-scope: out-of-scope
       restartPolicy: OnFailure
       securityContext:
-        fsGroup: 2000
         runAsNonRoot: true
-        runAsUser: 2000
+        runAsUser: 65532
+        fsGroup: 2000
       serviceAccountName: ingress-nginx-admission
 ---
 apiVersion: batch/v1
@@ -636,7 +649,7 @@ metadata:
     app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
-    app.kubernetes.io/version: 1.9.0
+    app.kubernetes.io/version: 1.11.3
     pci-scope: out-of-scope
   namespace: ingress-nginx
   name: ingress-nginx-admission-patch
@@ -649,7 +662,7 @@ spec:
         app.kubernetes.io/instance: ingress-nginx
         app.kubernetes.io/name: ingress-nginx
         app.kubernetes.io/part-of: ingress-nginx
-        app.kubernetes.io/version: 1.9.0
+        app.kubernetes.io/version: 1.11.3
         pci-scope: out-of-scope
     spec:
       containers:
@@ -665,11 +678,17 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: YOUR_ACR.azurecr.io/live/ingress-nginx/kube-webhook-certgen:v20230407
+        image: YOUR_ACR.azurecr.io/live/ingress-nginx/kube-webhook-certgen:v1.4.4
         imagePullPolicy: IfNotPresent
         name: patch
         securityContext:
           allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          seccompProfile:
+            type: RuntimeDefault
         resources:
           requests:
             cpu: 100m
@@ -683,6 +702,6 @@ spec:
       restartPolicy: OnFailure
       securityContext:
         runAsNonRoot: true
-        runAsUser: 2000
+        runAsUser: 65532
         fsGroup: 2000
       serviceAccountName: ingress-nginx-admission
diff --git a/docs/deploy/09-pre-cluster-stamp.md b/docs/deploy/09-pre-cluster-stamp.md
@@ -82,8 +82,8 @@ Using a security agent that is container-aware and can operate from within the c
    az acr import --source docker.io/falcosecurity/falcoctl:0.6.2 -t quarantine/falcosecurity/falcoctl:0.6.2 -n $ACR_NAME_QUARANTINE && \
    az acr import --source docker.io/library/busybox:1.36.1 -t quarantine/library/busybox:1.36.1 -n $ACR_NAME_QUARANTINE                                     && \
    az acr import --source ghcr.io/kubereboot/kured:1.14.0 -t quarantine/kubereboot/kured:1.14.0 -n $ACR_NAME_QUARANTINE                                     && \
-   az acr import --source registry.k8s.io/ingress-nginx/controller:v1.9.0 -t quarantine/ingress-nginx/controller:v1.9.0 -n $ACR_NAME_QUARANTINE             && \
-   az acr import --source registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20230407 -t quarantine/ingress-nginx/kube-webhook-certgen:v20230407 -n $ACR_NAME_QUARANTINE
+   az acr import --source registry.k8s.io/ingress-nginx/controller:v1.11.3 -t quarantine/ingress-nginx/controller:v1.11.3 -n $ACR_NAME_QUARANTINE && \
+   az acr import --source registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.4.4 -t quarantine/ingress-nginx/kube-webhook-certgen:v1.4.4 -n $ACR_NAME_QUARANTINE
    ```
 
    > The above imports account for 100% of the containers that you are actively bringing to the cluster, but not those that come with the AKS service itself nor any of its add-ons or extensions. Those images, outside of your direct control, are all sourced from Microsoft Container Registry's (MCR). While you do not have an affordance to inject yourself in the middle of their distribution to your cluster, you can still pull those images through your inspection process for your own audit and reporting purposes. *All container images that you directly bring to the cluster should pass through your quarantine process.* The *allowed images* Azure Policy associated with this cluster should be configured to match your specific needs. Be sure to update `allowedContainerImagesRegex` in [`cluster-stamp.json`](../../cluster-stamp.json) to define expected image sources to whatever specificity is manageable for you. Never allow a source that you do not intend to use. For example, if you do not bring Open Service Mesh into your cluster, you can remove the existing allowance for `mcr.microsoft.com` as a valid source of images, leaving just `<your acr instance>/live/` repositories as the only valid source for non-system namespaces.
@@ -122,8 +122,8 @@ Using a security agent that is container-aware and can operate from within the c
    az acr import --source quarantine/falcosecurity/falcoctl:0.6.2 -r $ACR_NAME_QUARANTINE -t live/falcosecurity/falcoctl:0.6.2 -n $ACR_NAME && \
    az acr import --source quarantine/library/busybox:1.36.1 -r $ACR_NAME_QUARANTINE -t live/library/busybox:1.36.1 -n $ACR_NAME                                     && \
    az acr import --source quarantine/kubereboot/kured:1.14.0 -r $ACR_NAME_QUARANTINE -t live/kubereboot/kured:1.14.0 -n $ACR_NAME                                   && \
-   az acr import --source quarantine/ingress-nginx/controller:v1.9.0 -r $ACR_NAME_QUARANTINE -t live/ingress-nginx/controller:v1.9.0 -n $ACR_NAME                   && \
-   az acr import --source quarantine/ingress-nginx/kube-webhook-certgen:v20230407 -r $ACR_NAME_QUARANTINE -t live/ingress-nginx/kube-webhook-certgen:v20230407 -n $ACR_NAME
+   az acr import --source quarantine/ingress-nginx/controller:v1.11.3 -r $ACR_NAME_QUARANTINE -t live/ingress-nginx/controller:v1.11.3 -n $ACR_NAME && \
+   az acr import --source quarantine/ingress-nginx/kube-webhook-certgen:v1.4.4 -r $ACR_NAME_QUARANTINE -t live/ingress-nginx/kube-webhook-certgen:v1.4.4 -n $ACR_NAME
    ```
 
 1. Trigger quarantine violation. *Optional.*

diff --git a/docs/deploy/13-validation.md b/docs/deploy/13-validation.md
@@ -11,7 +11,7 @@ This section will help you to validate the workload is exposed correctly and res
 1. Get the public IP of Azure Application Gateway.
 
    ```bash
-   APPGW_PUBLIC_IP=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-01 --query properties.outputs.appGwPublicIpAddress.value -o tsv)
+   APPGW_PUBLIC_IP=$(az deployment group show -g rg-enterprise-networking-spokes-centralus -n spoke-BU0001A0005-01 --query properties.outputs.appGwPublicIpAddress.value -o tsv)
    ```
 
 1. Create a DNS `A` Record 🛑