diff --git a/api/v1alpha1/nginxingresscontroller_types.go b/api/v1alpha1/nginxingresscontroller_types.go index 590d2978..ef5b1b0e 100644 --- a/api/v1alpha1/nginxingresscontroller_types.go +++ b/api/v1alpha1/nginxingresscontroller_types.go @@ -61,6 +61,10 @@ type NginxIngressControllerSpec struct { // +optional DefaultBackendService *NICNamespacedName `json:"defaultBackendService,omitempty"` + // CustomHTTPErrors defines the error codes that the NginxIngressController should send to its default-backend in case of error. + // +optional + CustomHTTPErrors []int32 `json:"customHTTPErrors,omitempty"` + // Scaling defines configuration options for how the Ingress Controller scales // +optional Scaling *Scaling `json:"scaling,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 5dea1a4c..27de4152 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -143,6 +143,11 @@ func (in *NginxIngressControllerSpec) DeepCopyInto(out *NginxIngressControllerSp *out = new(NICNamespacedName) **out = **in } + if in.CustomHTTPErrors != nil { + in, out := &in.CustomHTTPErrors, &out.CustomHTTPErrors + *out = make([]int32, len(*in)) + copy(*out, *in) + } if in.Scaling != nil { in, out := &in.Scaling, &out.Scaling *out = new(Scaling) diff --git a/config/crd/bases/approuting.kubernetes.azure.com_nginxingresscontrollers.yaml b/config/crd/bases/approuting.kubernetes.azure.com_nginxingresscontrollers.yaml index 07d0d281..44e1e473 100644 --- a/config/crd/bases/approuting.kubernetes.azure.com_nginxingresscontrollers.yaml +++ b/config/crd/bases/approuting.kubernetes.azure.com_nginxingresscontrollers.yaml @@ -61,6 +61,13 @@ spec: x-kubernetes-validations: - message: Value is immutable rule: self == oldSelf + customHTTPErrors: + description: CustomHTTPErrors defines the error codes that the NginxIngressController + should send to its default-backend in case of error. + items: + format: int32 + type: integer + type: array defaultBackendService: description: DefaultBackendService defines the service that the NginxIngressController should default to when given HTTP traffic with not matching known diff --git a/pkg/controller/nginxingress/nginx_ingress_controller.go b/pkg/controller/nginxingress/nginx_ingress_controller.go index 72e6302e..09701dc7 100644 --- a/pkg/controller/nginxingress/nginx_ingress_controller.go +++ b/pkg/controller/nginxingress/nginx_ingress_controller.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/Azure/aks-app-routing-operator/pkg/controller/keyvault" "net/url" + "strconv" "time" approutingv1alpha1 "github.com/Azure/aks-app-routing-operator/api/v1alpha1" @@ -581,6 +582,17 @@ func ToNginxIngressConfig(nic *approutingv1alpha1.NginxIngressController, defaul nginxIng.DefaultBackendService = nic.Spec.DefaultBackendService.Namespace + "/" + nic.Spec.DefaultBackendService.Name } + if len(nic.Spec.CustomHTTPErrors) != 0 { + errStr := "" + for i, errCode := range nic.Spec.CustomHTTPErrors { + errStr += strconv.Itoa(int(errCode)) + if i+1 < len(nic.Spec.CustomHTTPErrors) { + errStr += "," + } + } + nginxIng.CustomHTTPErrors = errStr + } + return nginxIng } diff --git a/pkg/controller/nginxingress/nginx_ingress_controller_test.go b/pkg/controller/nginxingress/nginx_ingress_controller_test.go index d9d479c0..b83bfa1b 100644 --- a/pkg/controller/nginxingress/nginx_ingress_controller_test.go +++ b/pkg/controller/nginxingress/nginx_ingress_controller_test.go @@ -892,6 +892,9 @@ func TestToNginxIngressConfig(t *testing.T) { FakeDefaultSSLCertNoNamespace := getFakeDefaultSSLCert("fake", "") FakeDefaultBackend := approutingv1alpha1.NICNamespacedName{"fakename", "fakenamespace"} + FakeCustomErrors := []int32{404, 503} + SingleCustomError := []int32{404} + EmptyCustomErrors := []int32{} FakeCertWithForceSSLRedirectTrue := getFakeDefaultSSLCert("fake", "fakenamespace") FakeCertWithForceSSLRedirectTrue.ForceSSLRedirect = true @@ -1333,6 +1336,93 @@ func TestToNginxIngressConfig(t *testing.T) { TargetCPUUtilizationPercentage: balancedTargetCPUUtilization, }, }, + { + name: "default controller class with DefaultBackendService and CustomHTTPErrors", + nic: &approutingv1alpha1.NginxIngressController{ + TypeMeta: metav1.TypeMeta{ + APIVersion: approutingv1alpha1.GroupVersion.String(), + Kind: "NginxIngressController", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: DefaultNicName, + }, + Spec: approutingv1alpha1.NginxIngressControllerSpec{ + ControllerNamePrefix: DefaultNicResourceName, + IngressClassName: DefaultIcName, + DefaultBackendService: &FakeDefaultBackend, + CustomHTTPErrors: FakeCustomErrors, + }, + }, + want: manifests.NginxIngressConfig{ + ControllerClass: defaultCc, + ResourceName: DefaultNicResourceName, + IcName: DefaultIcName, + ServiceConfig: &manifests.ServiceConfig{}, + DefaultBackendService: FakeDefaultBackend.Namespace + "/" + FakeDefaultBackend.Name, + CustomHTTPErrors: "404,503", + MaxReplicas: defaultMaxReplicas, + MinReplicas: defaultMinReplicas, + TargetCPUUtilizationPercentage: balancedTargetCPUUtilization, + }, + }, + { + name: "default controller class with DefaultBackendService and a single CustomHTTPError", + nic: &approutingv1alpha1.NginxIngressController{ + TypeMeta: metav1.TypeMeta{ + APIVersion: approutingv1alpha1.GroupVersion.String(), + Kind: "NginxIngressController", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: DefaultNicName, + }, + Spec: approutingv1alpha1.NginxIngressControllerSpec{ + ControllerNamePrefix: DefaultNicResourceName, + IngressClassName: DefaultIcName, + DefaultBackendService: &FakeDefaultBackend, + CustomHTTPErrors: SingleCustomError, + }, + }, + want: manifests.NginxIngressConfig{ + ControllerClass: defaultCc, + ResourceName: DefaultNicResourceName, + IcName: DefaultIcName, + ServiceConfig: &manifests.ServiceConfig{}, + DefaultBackendService: FakeDefaultBackend.Namespace + "/" + FakeDefaultBackend.Name, + CustomHTTPErrors: "404", + MaxReplicas: defaultMaxReplicas, + MinReplicas: defaultMinReplicas, + TargetCPUUtilizationPercentage: balancedTargetCPUUtilization, + }, + }, + { + name: "default controller class with DefaultBackendService and empty custom errors", + nic: &approutingv1alpha1.NginxIngressController{ + TypeMeta: metav1.TypeMeta{ + APIVersion: approutingv1alpha1.GroupVersion.String(), + Kind: "NginxIngressController", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: DefaultNicName, + }, + Spec: approutingv1alpha1.NginxIngressControllerSpec{ + ControllerNamePrefix: DefaultNicResourceName, + IngressClassName: DefaultIcName, + DefaultBackendService: &FakeDefaultBackend, + CustomHTTPErrors: EmptyCustomErrors, + }, + }, + want: manifests.NginxIngressConfig{ + ControllerClass: defaultCc, + ResourceName: DefaultNicResourceName, + IcName: DefaultIcName, + ServiceConfig: &manifests.ServiceConfig{}, + DefaultBackendService: FakeDefaultBackend.Namespace + "/" + FakeDefaultBackend.Name, + CustomHTTPErrors: "", + MaxReplicas: defaultMaxReplicas, + MinReplicas: defaultMinReplicas, + TargetCPUUtilizationPercentage: balancedTargetCPUUtilization, + }, + }, } for _, c := range cases { diff --git a/pkg/manifests/fixtures/nginx/default_version/internal-with-custom-http-errors-cert-and-service.yaml b/pkg/manifests/fixtures/nginx/default_version/internal-with-custom-http-errors-cert-and-service.yaml new file mode 100644 index 00000000..eaec7d9d --- /dev/null +++ b/pkg/manifests/fixtures/nginx/default_version/internal-with-custom-http-errors-cert-and-service.yaml @@ -0,0 +1,462 @@ +apiVersion: v1 +kind: Namespace +metadata: + creationTimestamp: null + labels: + openservicemesh.io/monitored-by: osm + name: test-namespace +spec: {} +status: {} +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx-private +spec: + controller: test-controller-class +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - configmaps + verbs: + - update +- apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resourceNames: + - nginx + resources: + - leases + verbs: + - get + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx +subjects: +- kind: ServiceAccount + name: nginx + namespace: test-namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx +subjects: +- kind: ServiceAccount + name: nginx + namespace: test-namespace +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + externalTrafficPolicy: Local + ports: + - name: http + port: 80 + targetPort: http + - name: https + port: 443 + targetPort: https + - name: prometheus + port: 10254 + targetPort: prometheus + selector: + app: nginx + type: LoadBalancer +status: + loadBalancer: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + revisionHistoryLimit: 2 + selector: + matchLabels: + app: nginx + strategy: {} + template: + metadata: + annotations: + openservicemesh.io/sidecar-injection: disabled + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app: nginx + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: kubernetes.azure.com/mode + operator: In + values: + - system + weight: 100 + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - /nginx-ingress-controller + - --ingress-class=nginx-private + - --controller-class=test-controller-class + - --election-id=nginx + - --publish-service=$(POD_NAMESPACE)/nginx + - --configmap=$(POD_NAMESPACE)/nginx + - --enable-annotation-validation=true + - --default-ssl-certificate=fakesslnamespace/fakesslname + - --default-backend-service=fakebackendnamespace/fakebackendname + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: test-registry/oss/kubernetes/ingress/nginx-ingress-controller:v1.10.0 + livenessProbe: + failureThreshold: 6 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + name: controller + ports: + - containerPort: 80 + name: http + - containerPort: 443 + name: https + - containerPort: 10254 + name: prometheus + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 500m + memory: 127Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + runAsNonRoot: true + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + priorityClassName: system-cluster-critical + serviceAccountName: nginx + tolerations: + - key: CriticalAddonsOnly + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + app: nginx + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway +status: {} +--- +apiVersion: v1 +data: + allow-snippet-annotations: "true" + annotation-value-word-blocklist: load_module,lua_package,_by_lua,location,root,proxy_pass,serviceaccount,{,},' +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +--- +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx + targetCPUUtilizationPercentage: 80 +status: + currentReplicas: 0 + desiredReplicas: 0 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: nginx +status: + currentHealthy: 0 + desiredHealthy: 0 + disruptionsAllowed: 0 + expectedPods: 0 +--- diff --git a/pkg/manifests/fixtures/nginx/default_version/internal-with-custom-http-errors.yaml b/pkg/manifests/fixtures/nginx/default_version/internal-with-custom-http-errors.yaml new file mode 100644 index 00000000..115545f3 --- /dev/null +++ b/pkg/manifests/fixtures/nginx/default_version/internal-with-custom-http-errors.yaml @@ -0,0 +1,460 @@ +apiVersion: v1 +kind: Namespace +metadata: + creationTimestamp: null + labels: + openservicemesh.io/monitored-by: osm + name: test-namespace +spec: {} +status: {} +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx-private +spec: + controller: test-controller-class +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - configmaps + verbs: + - update +- apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resourceNames: + - nginx + resources: + - leases + verbs: + - get + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx +subjects: +- kind: ServiceAccount + name: nginx + namespace: test-namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx +subjects: +- kind: ServiceAccount + name: nginx + namespace: test-namespace +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + externalTrafficPolicy: Local + ports: + - name: http + port: 80 + targetPort: http + - name: https + port: 443 + targetPort: https + - name: prometheus + port: 10254 + targetPort: prometheus + selector: + app: nginx + type: LoadBalancer +status: + loadBalancer: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + revisionHistoryLimit: 2 + selector: + matchLabels: + app: nginx + strategy: {} + template: + metadata: + annotations: + openservicemesh.io/sidecar-injection: disabled + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app: nginx + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: kubernetes.azure.com/mode + operator: In + values: + - system + weight: 100 + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - /nginx-ingress-controller + - --ingress-class=nginx-private + - --controller-class=test-controller-class + - --election-id=nginx + - --publish-service=$(POD_NAMESPACE)/nginx + - --configmap=$(POD_NAMESPACE)/nginx + - --enable-annotation-validation=true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: test-registry/oss/kubernetes/ingress/nginx-ingress-controller:v1.10.0 + livenessProbe: + failureThreshold: 6 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + name: controller + ports: + - containerPort: 80 + name: http + - containerPort: 443 + name: https + - containerPort: 10254 + name: prometheus + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 500m + memory: 127Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + runAsNonRoot: true + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + priorityClassName: system-cluster-critical + serviceAccountName: nginx + tolerations: + - key: CriticalAddonsOnly + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + app: nginx + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway +status: {} +--- +apiVersion: v1 +data: + allow-snippet-annotations: "true" + annotation-value-word-blocklist: load_module,lua_package,_by_lua,location,root,proxy_pass,serviceaccount,{,},' +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +--- +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx + targetCPUUtilizationPercentage: 80 +status: + currentReplicas: 0 + desiredReplicas: 0 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: nginx +status: + currentHealthy: 0 + desiredHealthy: 0 + disruptionsAllowed: 0 + expectedPods: 0 +--- diff --git a/pkg/manifests/fixtures/nginx/v1.10.0/internal-with-custom-http-errors-cert-and-service.yaml b/pkg/manifests/fixtures/nginx/v1.10.0/internal-with-custom-http-errors-cert-and-service.yaml new file mode 100644 index 00000000..eaec7d9d --- /dev/null +++ b/pkg/manifests/fixtures/nginx/v1.10.0/internal-with-custom-http-errors-cert-and-service.yaml @@ -0,0 +1,462 @@ +apiVersion: v1 +kind: Namespace +metadata: + creationTimestamp: null + labels: + openservicemesh.io/monitored-by: osm + name: test-namespace +spec: {} +status: {} +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx-private +spec: + controller: test-controller-class +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - configmaps + verbs: + - update +- apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resourceNames: + - nginx + resources: + - leases + verbs: + - get + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx +subjects: +- kind: ServiceAccount + name: nginx + namespace: test-namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx +subjects: +- kind: ServiceAccount + name: nginx + namespace: test-namespace +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + externalTrafficPolicy: Local + ports: + - name: http + port: 80 + targetPort: http + - name: https + port: 443 + targetPort: https + - name: prometheus + port: 10254 + targetPort: prometheus + selector: + app: nginx + type: LoadBalancer +status: + loadBalancer: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + revisionHistoryLimit: 2 + selector: + matchLabels: + app: nginx + strategy: {} + template: + metadata: + annotations: + openservicemesh.io/sidecar-injection: disabled + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app: nginx + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: kubernetes.azure.com/mode + operator: In + values: + - system + weight: 100 + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - /nginx-ingress-controller + - --ingress-class=nginx-private + - --controller-class=test-controller-class + - --election-id=nginx + - --publish-service=$(POD_NAMESPACE)/nginx + - --configmap=$(POD_NAMESPACE)/nginx + - --enable-annotation-validation=true + - --default-ssl-certificate=fakesslnamespace/fakesslname + - --default-backend-service=fakebackendnamespace/fakebackendname + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: test-registry/oss/kubernetes/ingress/nginx-ingress-controller:v1.10.0 + livenessProbe: + failureThreshold: 6 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + name: controller + ports: + - containerPort: 80 + name: http + - containerPort: 443 + name: https + - containerPort: 10254 + name: prometheus + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 500m + memory: 127Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + runAsNonRoot: true + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + priorityClassName: system-cluster-critical + serviceAccountName: nginx + tolerations: + - key: CriticalAddonsOnly + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + app: nginx + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway +status: {} +--- +apiVersion: v1 +data: + allow-snippet-annotations: "true" + annotation-value-word-blocklist: load_module,lua_package,_by_lua,location,root,proxy_pass,serviceaccount,{,},' +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +--- +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx + targetCPUUtilizationPercentage: 80 +status: + currentReplicas: 0 + desiredReplicas: 0 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: nginx +status: + currentHealthy: 0 + desiredHealthy: 0 + disruptionsAllowed: 0 + expectedPods: 0 +--- diff --git a/pkg/manifests/fixtures/nginx/v1.10.0/internal-with-custom-http-errors.yaml b/pkg/manifests/fixtures/nginx/v1.10.0/internal-with-custom-http-errors.yaml new file mode 100644 index 00000000..115545f3 --- /dev/null +++ b/pkg/manifests/fixtures/nginx/v1.10.0/internal-with-custom-http-errors.yaml @@ -0,0 +1,460 @@ +apiVersion: v1 +kind: Namespace +metadata: + creationTimestamp: null + labels: + openservicemesh.io/monitored-by: osm + name: test-namespace +spec: {} +status: {} +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx-private +spec: + controller: test-controller-class +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - configmaps + verbs: + - update +- apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resourceNames: + - nginx + resources: + - leases + verbs: + - get + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx +subjects: +- kind: ServiceAccount + name: nginx + namespace: test-namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx +subjects: +- kind: ServiceAccount + name: nginx + namespace: test-namespace +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + externalTrafficPolicy: Local + ports: + - name: http + port: 80 + targetPort: http + - name: https + port: 443 + targetPort: https + - name: prometheus + port: 10254 + targetPort: prometheus + selector: + app: nginx + type: LoadBalancer +status: + loadBalancer: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + revisionHistoryLimit: 2 + selector: + matchLabels: + app: nginx + strategy: {} + template: + metadata: + annotations: + openservicemesh.io/sidecar-injection: disabled + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app: nginx + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: kubernetes.azure.com/mode + operator: In + values: + - system + weight: 100 + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - /nginx-ingress-controller + - --ingress-class=nginx-private + - --controller-class=test-controller-class + - --election-id=nginx + - --publish-service=$(POD_NAMESPACE)/nginx + - --configmap=$(POD_NAMESPACE)/nginx + - --enable-annotation-validation=true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: test-registry/oss/kubernetes/ingress/nginx-ingress-controller:v1.10.0 + livenessProbe: + failureThreshold: 6 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + name: controller + ports: + - containerPort: 80 + name: http + - containerPort: 443 + name: https + - containerPort: 10254 + name: prometheus + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 500m + memory: 127Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + runAsNonRoot: true + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + priorityClassName: system-cluster-critical + serviceAccountName: nginx + tolerations: + - key: CriticalAddonsOnly + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + app: nginx + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway +status: {} +--- +apiVersion: v1 +data: + allow-snippet-annotations: "true" + annotation-value-word-blocklist: load_module,lua_package,_by_lua,location,root,proxy_pass,serviceaccount,{,},' +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +--- +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx + targetCPUUtilizationPercentage: 80 +status: + currentReplicas: 0 + desiredReplicas: 0 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: nginx +status: + currentHealthy: 0 + desiredHealthy: 0 + disruptionsAllowed: 0 + expectedPods: 0 +--- diff --git a/pkg/manifests/fixtures/nginx/v1.11.2/internal-with-custom-http-errors-cert-and-service.yaml b/pkg/manifests/fixtures/nginx/v1.11.2/internal-with-custom-http-errors-cert-and-service.yaml new file mode 100644 index 00000000..eaec7d9d --- /dev/null +++ b/pkg/manifests/fixtures/nginx/v1.11.2/internal-with-custom-http-errors-cert-and-service.yaml @@ -0,0 +1,462 @@ +apiVersion: v1 +kind: Namespace +metadata: + creationTimestamp: null + labels: + openservicemesh.io/monitored-by: osm + name: test-namespace +spec: {} +status: {} +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx-private +spec: + controller: test-controller-class +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - configmaps + verbs: + - update +- apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resourceNames: + - nginx + resources: + - leases + verbs: + - get + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx +subjects: +- kind: ServiceAccount + name: nginx + namespace: test-namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx +subjects: +- kind: ServiceAccount + name: nginx + namespace: test-namespace +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + externalTrafficPolicy: Local + ports: + - name: http + port: 80 + targetPort: http + - name: https + port: 443 + targetPort: https + - name: prometheus + port: 10254 + targetPort: prometheus + selector: + app: nginx + type: LoadBalancer +status: + loadBalancer: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + revisionHistoryLimit: 2 + selector: + matchLabels: + app: nginx + strategy: {} + template: + metadata: + annotations: + openservicemesh.io/sidecar-injection: disabled + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app: nginx + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: kubernetes.azure.com/mode + operator: In + values: + - system + weight: 100 + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - /nginx-ingress-controller + - --ingress-class=nginx-private + - --controller-class=test-controller-class + - --election-id=nginx + - --publish-service=$(POD_NAMESPACE)/nginx + - --configmap=$(POD_NAMESPACE)/nginx + - --enable-annotation-validation=true + - --default-ssl-certificate=fakesslnamespace/fakesslname + - --default-backend-service=fakebackendnamespace/fakebackendname + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: test-registry/oss/kubernetes/ingress/nginx-ingress-controller:v1.10.0 + livenessProbe: + failureThreshold: 6 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + name: controller + ports: + - containerPort: 80 + name: http + - containerPort: 443 + name: https + - containerPort: 10254 + name: prometheus + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 500m + memory: 127Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + runAsNonRoot: true + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + priorityClassName: system-cluster-critical + serviceAccountName: nginx + tolerations: + - key: CriticalAddonsOnly + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + app: nginx + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway +status: {} +--- +apiVersion: v1 +data: + allow-snippet-annotations: "true" + annotation-value-word-blocklist: load_module,lua_package,_by_lua,location,root,proxy_pass,serviceaccount,{,},' +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +--- +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx + targetCPUUtilizationPercentage: 80 +status: + currentReplicas: 0 + desiredReplicas: 0 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: nginx +status: + currentHealthy: 0 + desiredHealthy: 0 + disruptionsAllowed: 0 + expectedPods: 0 +--- diff --git a/pkg/manifests/fixtures/nginx/v1.11.2/internal-with-custom-http-errors.yaml b/pkg/manifests/fixtures/nginx/v1.11.2/internal-with-custom-http-errors.yaml new file mode 100644 index 00000000..115545f3 --- /dev/null +++ b/pkg/manifests/fixtures/nginx/v1.11.2/internal-with-custom-http-errors.yaml @@ -0,0 +1,460 @@ +apiVersion: v1 +kind: Namespace +metadata: + creationTimestamp: null + labels: + openservicemesh.io/monitored-by: osm + name: test-namespace +spec: {} +status: {} +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx-private +spec: + controller: test-controller-class +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - configmaps + verbs: + - update +- apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resourceNames: + - nginx + resources: + - leases + verbs: + - get + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx +subjects: +- kind: ServiceAccount + name: nginx + namespace: test-namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx +subjects: +- kind: ServiceAccount + name: nginx + namespace: test-namespace +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + externalTrafficPolicy: Local + ports: + - name: http + port: 80 + targetPort: http + - name: https + port: 443 + targetPort: https + - name: prometheus + port: 10254 + targetPort: prometheus + selector: + app: nginx + type: LoadBalancer +status: + loadBalancer: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + revisionHistoryLimit: 2 + selector: + matchLabels: + app: nginx + strategy: {} + template: + metadata: + annotations: + openservicemesh.io/sidecar-injection: disabled + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app: nginx + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: kubernetes.azure.com/mode + operator: In + values: + - system + weight: 100 + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - /nginx-ingress-controller + - --ingress-class=nginx-private + - --controller-class=test-controller-class + - --election-id=nginx + - --publish-service=$(POD_NAMESPACE)/nginx + - --configmap=$(POD_NAMESPACE)/nginx + - --enable-annotation-validation=true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: test-registry/oss/kubernetes/ingress/nginx-ingress-controller:v1.10.0 + livenessProbe: + failureThreshold: 6 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + name: controller + ports: + - containerPort: 80 + name: http + - containerPort: 443 + name: https + - containerPort: 10254 + name: prometheus + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 500m + memory: 127Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + runAsNonRoot: true + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + priorityClassName: system-cluster-critical + serviceAccountName: nginx + tolerations: + - key: CriticalAddonsOnly + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + app: nginx + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway +status: {} +--- +apiVersion: v1 +data: + allow-snippet-annotations: "true" + annotation-value-word-blocklist: load_module,lua_package,_by_lua,location,root,proxy_pass,serviceaccount,{,},' +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +--- +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx + targetCPUUtilizationPercentage: 80 +status: + currentReplicas: 0 + desiredReplicas: 0 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingress-controller + app.kubernetes.io/managed-by: aks-app-routing-operator + app.kubernetes.io/name: nginx + name: nginx + namespace: test-namespace +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: nginx +status: + currentHealthy: 0 + desiredHealthy: 0 + disruptionsAllowed: 0 + expectedPods: 0 +--- diff --git a/pkg/manifests/nginx.go b/pkg/manifests/nginx.go index 72cf495a..916e4bb6 100644 --- a/pkg/manifests/nginx.go +++ b/pkg/manifests/nginx.go @@ -474,6 +474,10 @@ func newNginxIngressControllerConfigmap(conf *config.Config, ingressConfig *Ngin confMap.Data["force-ssl-redirect"] = "true" } + if ingressConfig.CustomHTTPErrors != "" { + confMap.Data["custom-http-errors"] = ingressConfig.CustomHTTPErrors + } + return confMap } diff --git a/pkg/manifests/nginx_test.go b/pkg/manifests/nginx_test.go index af6b4331..1ddea6e2 100644 --- a/pkg/manifests/nginx_test.go +++ b/pkg/manifests/nginx_test.go @@ -212,6 +212,61 @@ var ( TargetCPUUtilizationPercentage: 80, }, }, + { + Name: "internal-with-custom-http-errors", + Conf: &config.Config{ + NS: "test-namespace", + Registry: "test-registry", + MSIClientID: "test-msi-client-id", + TenantID: "test-tenant-id", + Cloud: "test-cloud", + Location: "test-location", + }, + Deploy: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-operator-deploy", + UID: "test-operator-deploy-uid", + }, + }, + IngConfig: &NginxIngressConfig{ + ControllerClass: "test-controller-class", + ResourceName: "nginx", + IcName: "nginx-private", + CustomHTTPErrors: "404,503", + MinReplicas: 2, + MaxReplicas: 100, + TargetCPUUtilizationPercentage: 80, + }, + }, + { + Name: "internal-with-custom-http-errors-cert-and-service", + Conf: &config.Config{ + NS: "test-namespace", + Registry: "test-registry", + MSIClientID: "test-msi-client-id", + TenantID: "test-tenant-id", + Cloud: "test-cloud", + Location: "test-location", + }, + Deploy: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-operator-deploy", + UID: "test-operator-deploy-uid", + }, + }, + IngConfig: &NginxIngressConfig{ + ControllerClass: "test-controller-class", + ResourceName: "nginx", + IcName: "nginx-private", + CustomHTTPErrors: "404,503", + DefaultSSLCertificate: "fakesslnamespace/fakesslname", + DefaultBackendService: "fakebackendnamespace/fakebackendname", + ForceSSLRedirect: true, + MinReplicas: 2, + MaxReplicas: 100, + TargetCPUUtilizationPercentage: 80, + }, + }, { Name: "internal-with-default-backend-service-and-ssl-cert", Conf: &config.Config{ diff --git a/pkg/manifests/types.go b/pkg/manifests/types.go index baf54bc8..d347b207 100644 --- a/pkg/manifests/types.go +++ b/pkg/manifests/types.go @@ -58,7 +58,8 @@ type NginxIngressConfig struct { ServiceConfig *ServiceConfig // service config that specifies details about the LB, defaults if nil ForceSSLRedirect bool // flag to sets all redirects to HTTPS if there is a default TLS certificate (requires DefaultSSLCertificate) DefaultSSLCertificate string // namespace/name used to create SSL certificate for the default HTTPS server (catch-all) - DefaultBackendService string + DefaultBackendService string // namespace/name used to determine default backend service for / and /healthz endpoints + CustomHTTPErrors string // error codes passed to the configmap to configure nginx to send traffic with the specified headers to its defaultbackend service in case of error MinReplicas int32 MaxReplicas int32 // TargetCPUUtilizationPercentage is the target average CPU utilization of the Ingress Controller diff --git a/testing/e2e/manifests/clientServer.go b/testing/e2e/manifests/clientServer.go index 9a209995..cdf5da42 100644 --- a/testing/e2e/manifests/clientServer.go +++ b/testing/e2e/manifests/clientServer.go @@ -21,14 +21,15 @@ var serverContents string var nonAlphanumericRegex = regexp.MustCompile(`[^a-zA-Z0-9 ]+`) -type testingResources struct { - Client *appsv1.Deployment - Server *appsv1.Deployment - Service *corev1.Service - Ingress *netv1.Ingress +type ClientServerResources struct { + Client *appsv1.Deployment + Server *appsv1.Deployment + Ingress *netv1.Ingress + Service *corev1.Service + AddedObjects []client.Object } -func (t testingResources) Objects() []client.Object { +func (t ClientServerResources) Objects() []client.Object { ret := []client.Object{ t.Client, t.Server, @@ -36,6 +37,8 @@ func (t testingResources) Objects() []client.Object { t.Ingress, } + ret = append(ret, t.AddedObjects...) + for _, obj := range ret { setGroupKindVersion(obj) } @@ -43,7 +46,7 @@ func (t testingResources) Objects() []client.Object { return ret } -func ClientAndServer(namespace, name, nameserver, keyvaultURI, host, tlsHost string) testingResources { +func ClientAndServer(namespace, name, nameserver, keyvaultURI, host, tlsHost string) ClientServerResources { name = nonAlphanumericRegex.ReplaceAllString(name, "") clientDeployment := newGoDeployment(clientContents, namespace, name+"-client") clientDeployment.Spec.Template.Annotations["openservicemesh.io/sidecar-injection"] = "disabled" @@ -152,7 +155,7 @@ func ClientAndServer(namespace, name, nameserver, keyvaultURI, host, tlsHost str delete(ingress.Annotations, "kubernetes.azure.com/tls-cert-keyvault-uri") } - return testingResources{ + return ClientServerResources{ Client: clientDeployment, Server: serverDeployment, Service: service, diff --git a/testing/e2e/manifests/customErrorsClientServer.go b/testing/e2e/manifests/customErrorsClientServer.go new file mode 100644 index 00000000..855d8db5 --- /dev/null +++ b/testing/e2e/manifests/customErrorsClientServer.go @@ -0,0 +1,321 @@ +package manifests + +import ( + _ "embed" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +//go:embed embedded/customErrorsClient.go +var ceClientContents string + +//go:embed embedded/404.html +var notFoundContents string + +//go:embed embedded/503.html +var unavailableContents string + +func CustomErrorsClientAndServer(namespace, name, nameserver, keyvaultURI, host, tlsHost, ingressClassName string, serviceName *string) ClientServerResources { + name = nonAlphanumericRegex.ReplaceAllString(name, "") + + // Client deployment + errorsClientDeployment := newGoDeployment(ceClientContents, namespace, name+"-ce-client") + errorsClientDeployment.Spec.Template.Annotations["openservicemesh.io/sidecar-injection"] = "disabled" + errorsClientDeployment.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{ + { + Name: "LIVE", + Value: "https://" + host + liveServicePath, + }, + { + Name: "DEAD", + Value: "https://" + host + deadServicePath, + }, + { + Name: "NOT_FOUND", + Value: "https://" + host + notFoundPath, + }, + { + Name: "NAMESERVER", + Value: nameserver, + }, + { + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}}, + }, + } + errorsClientDeployment.Spec.Template.Spec.Containers[0].ReadinessProbe = &corev1.Probe{ + FailureThreshold: 1, + InitialDelaySeconds: 1, + PeriodSeconds: 1, + SuccessThreshold: 1, + TimeoutSeconds: 5, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/", + Port: intstr.FromInt(8080), + Scheme: corev1.URISchemeHTTP, + }, + }, + } + + errorsServerName := name + "-nginx-errors-server" + errorsServerDeployment := + &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: errorsServerName, + Namespace: namespace, + Labels: map[string]string{ + ManagedByKey: ManagedByVal, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: to.Ptr(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": errorsServerName}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": errorsServerName, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "nginx-error-server", + Image: "registry.k8s.io/ingress-nginx/nginx-errors:v20230505@sha256:3600dcd1bbd0d05959bb01af4b272714e94d22d24a64e91838e7183c80e53f7f", + Ports: []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: 8080, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "custom-error-pages", + MountPath: "/www", + }, + }, + }}, + Volumes: []corev1.Volume{ + { + Name: "custom-error-pages", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "custom-error-pages", + }, + Items: []corev1.KeyToPath{ + {Key: "404", Path: "404.html"}, + {Key: "503", Path: "503.html"}, + }, + }, + }, + }, + }, + }, + }, + }, + } + + errorsServiceName := name + "-nginx-errors-service" + if serviceName != nil { + errorsServiceName = *serviceName + } + errorsService := + &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: errorsServiceName, + Namespace: namespace, + Annotations: map[string]string{ + ManagedByKey: ManagedByVal, + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{ + Name: "http", + Port: 80, + TargetPort: intstr.FromInt(8080), + }}, + Selector: map[string]string{ + "app": errorsServerName, + }, + }, + } + + liveService := + &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "live-service", + Namespace: namespace, + Annotations: map[string]string{ + ManagedByKey: ManagedByVal, + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{ + Name: "http", + Port: 5678, + TargetPort: intstr.FromInt(5678), + }}, + Selector: map[string]string{ + "app": "live", + }, + }, + } + + deadService := + &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "dead-service", + Namespace: namespace, + Annotations: map[string]string{ + ManagedByKey: ManagedByVal, + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{ + Name: "http", + Port: 8080, + TargetPort: intstr.FromInt(8080), + }}, + Selector: map[string]string{ + "app": "dead", + }, + }, + } + + customErrorPagesConfigMap := + &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-error-pages", + Namespace: namespace, + }, + Data: map[string]string{ + "404": notFoundContents, + "503": unavailableContents, + }, + } + + liveServicePod := + &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "live-app", + Labels: map[string]string{ + "app": "live", + }, + Namespace: namespace, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "live-app", + Image: "hashicorp/http-echo", + Args: []string{"-text=live service"}, + }, + }, + }, + } + + liveIngress := &netv1.Ingress{ + TypeMeta: metav1.TypeMeta{ + Kind: "Ingress", + APIVersion: "networking.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name + "-live-ingress", + Namespace: namespace, + Annotations: map[string]string{ + ManagedByKey: ManagedByVal, + "kubernetes.azure.com/tls-cert-keyvault-uri": keyvaultURI, + }, + }, + Spec: netv1.IngressSpec{ + IngressClassName: to.Ptr(ingressClassName), + Rules: []netv1.IngressRule{{ + Host: host, + IngressRuleValue: netv1.IngressRuleValue{ + HTTP: &netv1.HTTPIngressRuleValue{ + Paths: []netv1.HTTPIngressPath{ + { + Path: liveServicePath, + PathType: to.Ptr(netv1.PathTypePrefix), + Backend: netv1.IngressBackend{ + Service: &netv1.IngressServiceBackend{ + Name: "live-service", + Port: netv1.ServiceBackendPort{ + Number: 5678, + }, + }, + }, + }, + { + Path: deadServicePath, + PathType: to.Ptr(netv1.PathTypePrefix), + Backend: netv1.IngressBackend{ + Service: &netv1.IngressServiceBackend{ + Name: "dead-service", + Port: netv1.ServiceBackendPort{ + Number: 8080, + }, + }, + }, + }, + }, + }, + }, + }}, + TLS: []netv1.IngressTLS{{ + Hosts: []string{tlsHost}, + SecretName: "keyvault-" + name + "-ingress", + }}, + }, + } + + if tlsHost == "" { + liveIngress.Spec.Rules[0].Host = "" + liveIngress.Spec.TLS = nil + delete(liveIngress.Annotations, "kubernetes.azure.com/tls-cert-keyvault-uri") + } + + return ClientServerResources{ + Client: errorsClientDeployment, + Server: errorsServerDeployment, + Ingress: liveIngress, + Service: errorsService, + AddedObjects: []client.Object{ + liveService, + deadService, + liveServicePod, + customErrorPagesConfigMap, + }, + } +} diff --git a/testing/e2e/manifests/defaultBackendClientServer.go b/testing/e2e/manifests/defaultBackendClientServer.go index 755f84ce..9a72cfab 100644 --- a/testing/e2e/manifests/defaultBackendClientServer.go +++ b/testing/e2e/manifests/defaultBackendClientServer.go @@ -3,7 +3,6 @@ package manifests import ( _ "embed" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -12,42 +11,22 @@ import ( ) //go:embed embedded/defaultBackendClient.go -var dbeClientContents string +var dbClientContents string //go:embed embedded/defaultBackendServer.go -var dbeServerContents string - -type ClientServerResources struct { - Client *appsv1.Deployment - Server *appsv1.Deployment - Ingress *netv1.Ingress - Service *corev1.Service - AddedObjects []client.Object -} - -func (t ClientServerResources) Objects() []client.Object { - ret := []client.Object{ - t.Client, - t.Server, - t.Service, - t.Ingress, - } - - ret = append(ret, t.AddedObjects...) - - for _, obj := range ret { - setGroupKindVersion(obj) - } - - return ret -} +var dbServerContents string + +var ( + validUrlPath = "/test" + invalidUrlPath = "/fakehost" + liveServicePath = "/live" + deadServicePath = "/dead" + notFoundPath = "/notfound" +) func DefaultBackendClientAndServer(namespace, name, nameserver, keyvaultURI, ingressClassName, host, tlsHost string) ClientServerResources { - validUrlPath := "/test" - invalidUrlPath := "/fakehost" - // Client deployment - clientDeployment := newGoDeployment(dbeClientContents, namespace, name+"-dbe-client") + clientDeployment := newGoDeployment(dbClientContents, namespace, name+"-db-client") clientDeployment.Spec.Template.Annotations["openservicemesh.io/sidecar-injection"] = "disabled" clientDeployment.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{ { @@ -83,8 +62,8 @@ func DefaultBackendClientAndServer(namespace, name, nameserver, keyvaultURI, ing } // Main server deployment - serverName := name + "-server" - serviceName := name + "-service" + serverName := name + "dbtest-server" + serviceName := name + "dbtest-service" serverDeployment := newGoDeployment(serverContents, namespace, serverName) ingressName := name + "-ingress" @@ -118,8 +97,8 @@ func DefaultBackendClientAndServer(namespace, name, nameserver, keyvaultURI, ing defaultServerName := "default-" + name + "-server" defaultServiceName := "default-" + name + "-service" - defaultServerDeployment := newGoDeployment(dbeServerContents, namespace, defaultServerName) - dbeService := + defaultServerDeployment := newGoDeployment(dbServerContents, namespace, defaultServerName) + dbService := &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -198,7 +177,7 @@ func DefaultBackendClientAndServer(namespace, name, nameserver, keyvaultURI, ing Service: service, AddedObjects: []client.Object{ defaultServerDeployment, - dbeService, + dbService, }, } } diff --git a/testing/e2e/manifests/embedded/404.html b/testing/e2e/manifests/embedded/404.html new file mode 100644 index 00000000..281b8982 --- /dev/null +++ b/testing/e2e/manifests/embedded/404.html @@ -0,0 +1 @@ +CONFIRMING CUSTOM 404 TEST MESSAGE \ No newline at end of file diff --git a/testing/e2e/manifests/embedded/503.html b/testing/e2e/manifests/embedded/503.html new file mode 100644 index 00000000..aa5ab31a --- /dev/null +++ b/testing/e2e/manifests/embedded/503.html @@ -0,0 +1 @@ +CONFIRMING CUSTOM 503 TEST MESSAGE \ No newline at end of file diff --git a/testing/e2e/manifests/embedded/customErrorsClient.go b/testing/e2e/manifests/embedded/customErrorsClient.go new file mode 100644 index 00000000..7c5ff34a --- /dev/null +++ b/testing/e2e/manifests/embedded/customErrorsClient.go @@ -0,0 +1,123 @@ +package main + +import ( + "context" + "crypto/tls" + "io/ioutil" + "log" + "net" + "net/http" + "os" + "regexp" + "time" +) + +func main() { + nameserver := os.Getenv("NAMESERVER") + + dialer := &net.Dialer{Resolver: &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{Timeout: time.Second} + var ns string + if nameserver[:len(nameserver)-1] == "." { + ns = nameserver[:len(ns)-1] // remove trailing period added for some reason by azure dns + } else { + ns = nameserver // no need to remove trailing period if single entry coming from k8s vnet ns server + } + + return d.DialContext(ctx, "tcp", ns+":53") + }, + }} + client := &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + DialContext: dialer.DialContext, + }} + + liveServiceBody := "live service" + deadServiceBody := "CONFIRMING CUSTOM 503 TEST MESSAGE" + notFoundBody := "CONFIRMING CUSTOM 404 TEST MESSAGE" + + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + // live service tests + liveResp, err := client.Get(os.Getenv("LIVE")) + if err != nil { + log.Printf("error sending request: %s", err) + w.WriteHeader(500) + return + } + defer liveResp.Body.Close() + + body, err := ioutil.ReadAll(liveResp.Body) + + if err != nil { + log.Printf("error reading response body: %s", err) + w.WriteHeader(500) + return + } + + cleanedBody := cleanBody(string(body)[:(len(liveServiceBody))]) + log.Printf("received response: %s from url %s", cleanedBody, os.Getenv("LIVE")) + if cleanedBody != liveServiceBody { + log.Printf("unexpected response body: %s, expected: %s", cleanedBody, liveServiceBody) + w.WriteHeader(500) + return + } + + // dead service tests - 503 error codes + deadResp, err := client.Get(os.Getenv("DEAD")) + if err != nil { + log.Printf("error sending request: %s", err) + w.WriteHeader(500) + return + } + defer deadResp.Body.Close() + + body, err = ioutil.ReadAll(deadResp.Body) + if err != nil { + log.Printf("error reading response body: %s", err) + w.WriteHeader(500) + return + } + + cleanedBody = cleanBody(string(body)) + log.Printf("received response %s from url %s", cleanedBody, os.Getenv("DEAD")) + if cleanedBody != deadServiceBody { + log.Printf("unexpected response body: %s expected: %s", cleanedBody, deadServiceBody) + w.WriteHeader(500) + return + } + + // not found tests - 404 error codes + notFoundResp, err := client.Get(os.Getenv("NOT_FOUND")) + if err != nil { + log.Printf("error sending request: %s", err) + w.WriteHeader(500) + return + } + defer notFoundResp.Body.Close() + + body, err = ioutil.ReadAll(notFoundResp.Body) + if err != nil { + log.Printf("error reading response body: %s", err) + w.WriteHeader(500) + return + } + + cleanedBody = cleanBody(string(body)) + log.Printf("received response %s from url %s", cleanedBody, os.Getenv("TEST_URL")) + if cleanedBody != notFoundBody { + log.Printf("unexpected response body: %s expected: %s", cleanedBody, notFoundBody) + w.WriteHeader(500) + return + } + }) + panic(http.ListenAndServe(":8080", nil)) +} + +func cleanBody(body string) string { + pattern := "/(?:\\r\\n|\\r|\\n)/g" + cleaner := regexp.MustCompile(pattern) + + return cleaner.ReplaceAllString(body, "") +} diff --git a/testing/e2e/suites/defaultBackendService.go b/testing/e2e/suites/defaultBackendService.go index 02b7be27..8ef9a46d 100644 --- a/testing/e2e/suites/defaultBackendService.go +++ b/testing/e2e/suites/defaultBackendService.go @@ -24,87 +24,150 @@ import ( ) var ( - dbeScheme = runtime.NewScheme() - dbeBasicNS = make(map[string]*corev1.Namespace) - dbeServiceName = "dbeservice" + dbScheme = runtime.NewScheme() + dbBasicNS = make(map[string]*corev1.Namespace) + ceBasicNS = make(map[string]*corev1.Namespace) nonAlphaNumHyphenRegex = regexp.MustCompile(`[^a-zA-Z0-9- ]+`) trailingHyphenRegex = regexp.MustCompile(`^-+|-+$`) ) func init() { - netv1.AddToScheme(dbeScheme) - v1alpha1.AddToScheme(dbeScheme) - batchv1.AddToScheme(dbeScheme) - corev1.AddToScheme(dbeScheme) - metav1.AddMetaToScheme(dbeScheme) - appsv1.AddToScheme(dbeScheme) - policyv1.AddToScheme(dbeScheme) - rbacv1.AddToScheme(dbeScheme) - secv1.AddToScheme(dbeScheme) + netv1.AddToScheme(dbScheme) + v1alpha1.AddToScheme(dbScheme) + batchv1.AddToScheme(dbScheme) + corev1.AddToScheme(dbScheme) + metav1.AddMetaToScheme(dbScheme) + appsv1.AddToScheme(dbScheme) + policyv1.AddToScheme(dbScheme) + rbacv1.AddToScheme(dbScheme) + secv1.AddToScheme(dbScheme) } func defaultBackendTests(in infra.Provisioned) []test { - return []test{{ - name: "testing default backend service validity", - cfgs: builderFromInfra(in). - withOsm(in, false, true). - withVersions(manifests.OperatorVersionLatest). - withZones(manifests.AllDnsZoneCounts, manifests.AllDnsZoneCounts). - build(), - run: func(ctx context.Context, config *rest.Config, operator manifests.OperatorConfig) error { - lgr := logger.FromContext(ctx) - lgr.Info("starting test") - - c, err := client.New(config, client.Options{ - Scheme: dbeScheme, - }) - if err != nil { - return fmt.Errorf("creating client: %w", err) - } + return []test{ + { + name: "testing default backend service validity", + cfgs: builderFromInfra(in). + withOsm(in, false, true). + withVersions(manifests.OperatorVersionLatest). + withZones(manifests.AllDnsZoneCounts, manifests.AllDnsZoneCounts). + build(), + run: func(ctx context.Context, config *rest.Config, operator manifests.OperatorConfig) error { + lgr := logger.FromContext(ctx) + lgr.Info("starting test") + + c, err := client.New(config, client.Options{ + Scheme: dbScheme, + }) + if err != nil { + return fmt.Errorf("creating client: %w", err) + } - ingressClassName := "dbeingressclass" - nic := &v1alpha1.NginxIngressController{ - TypeMeta: metav1.TypeMeta{ - Kind: "NginxIngressController", - APIVersion: "approuting.kubernetes.azure.com/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "default-nginxingress", - Annotations: map[string]string{ - manifests.ManagedByKey: manifests.ManagedByVal, + ingressClassName := "dbingressclass" + nic := &v1alpha1.NginxIngressController{ + TypeMeta: metav1.TypeMeta{ + Kind: "NginxIngressController", + APIVersion: "approuting.kubernetes.azure.com/v1alpha1", }, - }, - Spec: v1alpha1.NginxIngressControllerSpec{ - IngressClassName: ingressClassName, - ControllerNamePrefix: "nginx-default-backend", - }, - } + ObjectMeta: metav1.ObjectMeta{ + Name: "db-nginxingress", + Annotations: map[string]string{ + manifests.ManagedByKey: manifests.ManagedByVal, + }, + }, + Spec: v1alpha1.NginxIngressControllerSpec{ + IngressClassName: ingressClassName, + ControllerNamePrefix: "nginx-default-backend", + }, + } - if err := upsert(ctx, c, nic); err != nil { - return fmt.Errorf("upserting nic: %w", err) - } + if err := upsert(ctx, c, nic); err != nil { + return fmt.Errorf("upserting nic: %w", err) + } - var service = &v1alpha1.ManagedObjectReference{} - lgr.Info("checking for service in managed resource refs") - for _, ref := range nic.Status.ManagedResourceRefs { - if ref.Kind == "Service" { - lgr.Info("found service") - service = &ref + var service = &v1alpha1.ManagedObjectReference{} + lgr.Info("checking for service in managed resource refs") + for _, ref := range nic.Status.ManagedResourceRefs { + if ref.Kind == "Service" { + lgr.Info("found service") + service = &ref + } } - } - if service == nil { - return fmt.Errorf("no service available in resource refs") - } + if service == nil { + return fmt.Errorf("no service available in resource refs") + } - if err := defaultBackendClientServerTest(ctx, config, operator, dbeBasicNS, in, to.Ptr(service.Name), c, ingressClassName, nic); err != nil { - return err - } + if err := defaultBackendClientServerTest(ctx, config, operator, dbBasicNS, in, to.Ptr(service.Name), c, ingressClassName, nic); err != nil { + return err + } - lgr.Info("finished testing") - return nil + lgr.Info("finished testing") + return nil + }, + }, + { + name: "testing custom http error validity", + cfgs: builderFromInfra(in). + withOsm(in, false, true). + withVersions(manifests.OperatorVersionLatest). + withZones(manifests.AllDnsZoneCounts, manifests.AllDnsZoneCounts). + build(), + run: func(ctx context.Context, config *rest.Config, operator manifests.OperatorConfig) error { + lgr := logger.FromContext(ctx) + lgr.Info("starting custom errors test") + + c, err := client.New(config, client.Options{ + Scheme: dbScheme, + }) + if err != nil { + return fmt.Errorf("creating client: %w", err) + } + + ingressClassName := "ceingressclass" + nic := + &v1alpha1.NginxIngressController{ + TypeMeta: metav1.TypeMeta{ + Kind: "NginxIngressController", + APIVersion: "approuting.kubernetes.azure.com/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ce-nginxingress", + Annotations: map[string]string{ + manifests.ManagedByKey: manifests.ManagedByVal, + }, + }, + Spec: v1alpha1.NginxIngressControllerSpec{ + IngressClassName: ingressClassName, + ControllerNamePrefix: "nginx-custom-errors", + CustomHTTPErrors: []int32{404, 503}, + }, + } + if err := upsert(ctx, c, nic); err != nil { + return fmt.Errorf("upserting nic: %w", err) + } + + var service = &v1alpha1.ManagedObjectReference{} + lgr.Info("checking for service in managed resource refs") + for _, ref := range nic.Status.ManagedResourceRefs { + if ref.Kind == "Service" { + lgr.Info("found service") + service = &ref + } + } + + if service == nil { + return fmt.Errorf("no service available in resource refs") + } + + if err := defaultBackendClientServerTest(ctx, config, operator, ceBasicNS, in, to.Ptr(service.Name), c, ingressClassName, nic); err != nil { + return err + } + + lgr.Info("finished testing") + return nil + }, }, - }, } } @@ -207,8 +270,13 @@ var defaultBackendClientServerTest = func(ctx context.Context, config *rest.Conf } } - testingResources = manifests.DefaultBackendClientAndServer(zoneNamespace, zoneName, zone.GetNameserver(), zoneKVUri, ingressClassName, zoneHost, tlsHost) - nic.Spec.DefaultBackendService = &v1alpha1.NICNamespacedName{"default-" + zoneName + "-service", zoneNamespace} + if nic.Spec.CustomHTTPErrors != nil && len(nic.Spec.CustomHTTPErrors) > 1 { + testingResources = manifests.CustomErrorsClientAndServer(zoneNamespace, zoneName, zone.GetNameserver(), zoneKVUri, zoneHost, tlsHost, ingressClassName, serviceName) + nic.Spec.DefaultBackendService = &v1alpha1.NICNamespacedName{testingResources.Service.Name, testingResources.Service.Namespace} + } else { + testingResources = manifests.DefaultBackendClientAndServer(zoneNamespace, zoneName, zone.GetNameserver(), zoneKVUri, ingressClassName, zoneHost, tlsHost) + nic.Spec.DefaultBackendService = &v1alpha1.NICNamespacedName{"default-" + zoneName + "-service", zoneNamespace} + } upsertObjects = append(upsertObjects, testingResources.Objects()...) upsertObjects = append(upsertObjects, nic)