From b4dc2ac6ec988a57a05759a5f4de7c5cd87b336a Mon Sep 17 00:00:00 2001 From: Elliott Baron Date: Mon, 16 Sep 2024 16:01:19 -0400 Subject: [PATCH 1/6] feat(agent): add nginx proxy for agent communication --- Makefile | 4 + api/v1beta2/cryostat_types.go | 21 ++ api/v1beta2/zz_generated.deepcopy.go | 32 +++ ...yostat-operator.clusterserviceversion.yaml | 14 +- .../operator.cryostat.io_cryostats.yaml | 250 +++++++++++++++++ .../bases/operator.cryostat.io_cryostats.yaml | 250 +++++++++++++++++ config/default/image_tag_patch.yaml | 2 + ...yostat-operator.clusterserviceversion.yaml | 8 + hack/image_tag_patch.yaml.in | 2 + internal/controllers/certmanager.go | 10 +- .../resource_definitions/certificates.go | 24 ++ .../resource_definitions.go | 103 ++++++- internal/controllers/configmaps.go | 261 ++++++++++++++---- internal/controllers/const_generated.go | 3 + internal/controllers/constants/constants.go | 8 + internal/controllers/reconciler.go | 12 + internal/controllers/services.go | 61 ++++ internal/tools/const_generator.go | 42 +-- 18 files changed, 1036 insertions(+), 71 deletions(-) diff --git a/Makefile b/Makefile index d8522d72..4bf2ed26 100644 --- a/Makefile +++ b/Makefile @@ -93,6 +93,10 @@ STORAGE_NAMESPACE ?= $(DEFAULT_NAMESPACE) STORAGE_NAME ?= cryostat-storage STORAGE_VERSION ?= latest export STORAGE_IMG ?= $(STORAGE_NAMESPACE)/$(STORAGE_NAME):$(STORAGE_VERSION) +AGENT_PROXY_NAMESPACE ?= registry.access.redhat.com/ubi8 +AGENT_PROXY_NAME ?= nginx-124 +AGENT_PROXY_VERSION ?= latest +export AGENT_PROXY_IMG = $(AGENT_PROXY_NAMESPACE)/$(AGENT_PROXY_NAME):$(AGENT_PROXY_VERSION) CERT_MANAGER_VERSION ?= 1.11.5 CERT_MANAGER_MANIFEST ?= \ diff --git a/api/v1beta2/cryostat_types.go b/api/v1beta2/cryostat_types.go index 3d8bca8e..4148c579 100644 --- a/api/v1beta2/cryostat_types.go +++ b/api/v1beta2/cryostat_types.go @@ -143,6 +143,10 @@ type ResourceConfigList struct { // +optional // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:resourceRequirements"} ObjectStorageResources corev1.ResourceRequirements `json:"objectStorageResources,omitempty"` + // Resource requirements for the agent proxy container. + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:resourceRequirements"} + AgentProxyResources corev1.ResourceRequirements `json:"agentProxyResources,omitempty"` } // CryostatStatus defines the observed state of Cryostat. @@ -306,6 +310,16 @@ type ReportsServiceConfig struct { ServiceConfig `json:",inline"` } +// AgentServiceConfig provides customization for the service handling +// traffic from Cryostat agents to the Cryostat application. +type AgentServiceConfig struct { + // HTTP port number for the Cryostat agent API service. + // Defaults to 8443. + // +optional + HTTPPort *int32 `json:"httpPort,omitempty"` + ServiceConfig `json:",inline"` +} + // ServiceConfigList holds the service configuration for each // service created by the operator. type ServiceConfigList struct { @@ -315,6 +329,9 @@ type ServiceConfigList struct { // Specification for the service responsible for the cryostat-reports sidecars. // +optional ReportsConfig *ReportsServiceConfig `json:"reportsConfig,omitempty"` + // Specification for the service responsible for agents to communicate with Cryostat. + // +optional + AgentConfig *AgentServiceConfig `json:"agentConfig,omitempty"` } // NetworkConfiguration provides customization for how to expose a Cryostat @@ -567,6 +584,10 @@ type SecurityOptions struct { // +optional // +operator-sdk:csv:customresourcedefinitions:type=spec DatabaseSecurityContext *corev1.SecurityContext `json:"databaseSecurityContext,omitempty"` + // Security Context to apply to the agent proxy container. + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + AgentProxySecurityContext *corev1.SecurityContext `json:"agentProxySecurityContext,omitempty"` } // ReportsSecurityOptions contains Security Context customizations for the diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index fe589e9c..f5e241df 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -56,6 +56,27 @@ func (in *Affinity) DeepCopy() *Affinity { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AgentServiceConfig) DeepCopyInto(out *AgentServiceConfig) { + *out = *in + if in.HTTPPort != nil { + in, out := &in.HTTPPort, &out.HTTPPort + *out = new(int32) + **out = **in + } + in.ServiceConfig.DeepCopyInto(&out.ServiceConfig) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentServiceConfig. +func (in *AgentServiceConfig) DeepCopy() *AgentServiceConfig { + if in == nil { + return nil + } + out := new(AgentServiceConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AuthorizationOptions) DeepCopyInto(out *AuthorizationOptions) { *out = *in @@ -579,6 +600,7 @@ func (in *ResourceConfigList) DeepCopyInto(out *ResourceConfigList) { in.GrafanaResources.DeepCopyInto(&out.GrafanaResources) in.DatabaseResources.DeepCopyInto(&out.DatabaseResources) in.ObjectStorageResources.DeepCopyInto(&out.ObjectStorageResources) + in.AgentProxyResources.DeepCopyInto(&out.AgentProxyResources) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceConfigList. @@ -717,6 +739,11 @@ func (in *SecurityOptions) DeepCopyInto(out *SecurityOptions) { *out = new(corev1.SecurityContext) (*in).DeepCopyInto(*out) } + if in.AgentProxySecurityContext != nil { + in, out := &in.AgentProxySecurityContext, &out.AgentProxySecurityContext + *out = new(corev1.SecurityContext) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityOptions. @@ -776,6 +803,11 @@ func (in *ServiceConfigList) DeepCopyInto(out *ServiceConfigList) { *out = new(ReportsServiceConfig) (*in).DeepCopyInto(*out) } + if in.AgentConfig != nil { + in, out := &in.AgentConfig, &out.AgentConfig + *out = new(AgentServiceConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceConfigList. diff --git a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml index be770223..bc9fc802 100644 --- a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml +++ b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml @@ -30,7 +30,7 @@ metadata: capabilities: Seamless Upgrades categories: Monitoring, Developer Tools containerImage: quay.io/cryostat/cryostat-operator:4.0.0-dev - createdAt: "2024-09-11T17:33:08Z" + createdAt: "2024-09-19T13:48:41Z" description: JVM monitoring and profiling tool operatorframework.io/initialization-resource: |- { @@ -230,6 +230,11 @@ spec: - description: Resource requirements for the Cryostat deployment. displayName: Resources path: resources + - description: Resource requirements for the agent proxy container. + displayName: Agent Proxy Resources + path: resources.agentProxyResources + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:resourceRequirements - description: Resource requirements for the auth proxy. displayName: Auth Proxy Resources path: resources.authProxyResources @@ -294,6 +299,9 @@ spec: path: securityOptions x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced + - description: Security Context to apply to the agent proxy container. + displayName: Agent Proxy Security Context + path: securityOptions.agentProxySecurityContext - description: Security Context to apply to the auth proxy container. displayName: Auth Proxy Security Context path: securityOptions.authProxySecurityContext @@ -1010,6 +1018,8 @@ spec: value: quay.io/cryostat/cryostat-storage:latest - name: RELATED_IMAGE_DATABASE value: quay.io/cryostat/cryostat-db:latest + - name: RELATED_IMAGE_AGENT_PROXY + value: registry.access.redhat.com/ubi8/nginx-124:latest - name: WATCH_NAMESPACE valueFrom: fieldRef: @@ -1179,6 +1189,8 @@ spec: name: storage - image: quay.io/cryostat/cryostat-db:latest name: database + - image: registry.access.redhat.com/ubi8/nginx-124:latest + name: agent-proxy version: 4.0.0-dev webhookdefinitions: - admissionReviewVersions: diff --git a/bundle/manifests/operator.cryostat.io_cryostats.yaml b/bundle/manifests/operator.cryostat.io_cryostats.yaml index bda0a2b6..05092905 100644 --- a/bundle/manifests/operator.cryostat.io_cryostats.yaml +++ b/bundle/manifests/operator.cryostat.io_cryostats.yaml @@ -6571,6 +6571,61 @@ spec: resources: description: Resource requirements for the Cryostat deployment. properties: + agentProxyResources: + description: Resource requirements for the agent proxy container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object authProxyResources: description: Resource requirements for the auth proxy. properties: @@ -7724,6 +7779,173 @@ spec: description: Options to configure the Security Contexts for the Cryostat application. properties: + agentProxySecurityContext: + description: Security Context to apply to the agent proxy container. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default is DefaultProcMount which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object authProxySecurityContext: description: Security Context to apply to the auth proxy container. properties: @@ -8907,6 +9129,34 @@ spec: description: Options to customize the services created for the Cryostat application. properties: + agentConfig: + description: Specification for the service responsible for agents + to communicate with Cryostat. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the service during its + creation. + type: object + httpPort: + description: |- + HTTP port number for the Cryostat agent API service. + Defaults to 8443. + format: int32 + type: integer + labels: + additionalProperties: + type: string + description: |- + Labels to add to the service during its creation. + The labels with keys "app" and "component" are reserved + for use by the operator. + type: object + serviceType: + description: Type of service to create. Defaults to "ClusterIP". + type: string + type: object coreConfig: description: Specification for the service responsible for the Cryostat application. diff --git a/config/crd/bases/operator.cryostat.io_cryostats.yaml b/config/crd/bases/operator.cryostat.io_cryostats.yaml index 16ebe539..a963d40b 100644 --- a/config/crd/bases/operator.cryostat.io_cryostats.yaml +++ b/config/crd/bases/operator.cryostat.io_cryostats.yaml @@ -6558,6 +6558,61 @@ spec: resources: description: Resource requirements for the Cryostat deployment. properties: + agentProxyResources: + description: Resource requirements for the agent proxy container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object authProxyResources: description: Resource requirements for the auth proxy. properties: @@ -7711,6 +7766,173 @@ spec: description: Options to configure the Security Contexts for the Cryostat application. properties: + agentProxySecurityContext: + description: Security Context to apply to the agent proxy container. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default is DefaultProcMount which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object authProxySecurityContext: description: Security Context to apply to the auth proxy container. properties: @@ -8894,6 +9116,34 @@ spec: description: Options to customize the services created for the Cryostat application. properties: + agentConfig: + description: Specification for the service responsible for agents + to communicate with Cryostat. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to add to the service during its + creation. + type: object + httpPort: + description: |- + HTTP port number for the Cryostat agent API service. + Defaults to 8443. + format: int32 + type: integer + labels: + additionalProperties: + type: string + description: |- + Labels to add to the service during its creation. + The labels with keys "app" and "component" are reserved + for use by the operator. + type: object + serviceType: + description: Type of service to create. Defaults to "ClusterIP". + type: string + type: object coreConfig: description: Specification for the service responsible for the Cryostat application. diff --git a/config/default/image_tag_patch.yaml b/config/default/image_tag_patch.yaml index 262dcd19..5cf4f4c4 100644 --- a/config/default/image_tag_patch.yaml +++ b/config/default/image_tag_patch.yaml @@ -25,3 +25,5 @@ spec: value: "quay.io/cryostat/cryostat-storage:latest" - name: RELATED_IMAGE_DATABASE value: "quay.io/cryostat/cryostat-db:latest" + - name: RELATED_IMAGE_AGENT_PROXY + value: "registry.access.redhat.com/ubi8/nginx-124:latest" diff --git a/config/manifests/bases/cryostat-operator.clusterserviceversion.yaml b/config/manifests/bases/cryostat-operator.clusterserviceversion.yaml index 636add06..ec137f4b 100644 --- a/config/manifests/bases/cryostat-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/cryostat-operator.clusterserviceversion.yaml @@ -244,6 +244,11 @@ spec: - description: Resource requirements for the Cryostat deployment. displayName: Resources path: resources + - description: Resource requirements for the agent proxy container. + displayName: Agent Proxy Resources + path: resources.agentProxyResources + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:resourceRequirements - description: Resource requirements for the auth proxy. displayName: Auth Proxy Resources path: resources.authProxyResources @@ -312,6 +317,9 @@ spec: path: securityOptions x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced + - description: Security Context to apply to the agent proxy container. + displayName: Agent Proxy Security Context + path: securityOptions.agentProxySecurityContext - description: Security Context to apply to the auth proxy container. displayName: Auth Proxy Security Context path: securityOptions.authProxySecurityContext diff --git a/hack/image_tag_patch.yaml.in b/hack/image_tag_patch.yaml.in index 9664c5a4..a51d13d4 100644 --- a/hack/image_tag_patch.yaml.in +++ b/hack/image_tag_patch.yaml.in @@ -25,3 +25,5 @@ spec: value: "${STORAGE_IMG}" - name: RELATED_IMAGE_DATABASE value: "${DATABASE_IMG}" + - name: RELATED_IMAGE_AGENT_PROXY + value: "${AGENT_PROXY_IMG}" diff --git a/internal/controllers/certmanager.go b/internal/controllers/certmanager.go index 2fb79588..35df44c1 100644 --- a/internal/controllers/certmanager.go +++ b/internal/controllers/certmanager.go @@ -91,8 +91,15 @@ func (r *Reconciler) setupTLS(ctx context.Context, cr *model.CryostatInstance) ( return nil, err } + // Create a certificate for the agent proxy signed by the Cryostat CA + agentProxyCert := resources.NewAgentProxyCert(cr) + err = r.createOrUpdateCertificate(ctx, agentProxyCert, cr.Object) + if err != nil { + return nil, err + } + // List of certificates whose secrets should be owned by this CR - certificates := []*certv1.Certificate{caCert, cryostatCert, reportsCert} + certificates := []*certv1.Certificate{caCert, cryostatCert, reportsCert, agentProxyCert} // Get the Cryostat CA certificate bytes from certificate secret caBytes, err := r.getCertficateBytes(ctx, caCert) @@ -103,6 +110,7 @@ func (r *Reconciler) setupTLS(ctx context.Context, cr *model.CryostatInstance) ( tlsConfig := &resources.TLSConfig{ CryostatSecret: cryostatCert.Spec.SecretName, ReportsSecret: reportsCert.Spec.SecretName, + AgentProxySecret: agentProxyCert.Spec.SecretName, KeystorePassSecret: cryostatCert.Spec.Keystores.PKCS12.PasswordSecretRef.Name, CACert: caBytes, } diff --git a/internal/controllers/common/resource_definitions/certificates.go b/internal/controllers/common/resource_definitions/certificates.go index d2665c90..be757c74 100644 --- a/internal/controllers/common/resource_definitions/certificates.go +++ b/internal/controllers/common/resource_definitions/certificates.go @@ -155,3 +155,27 @@ func NewAgentCert(cr *model.CryostatInstance, namespace string, gvk *schema.Grou }, } } + +func NewAgentProxyCert(cr *model.CryostatInstance) *certv1.Certificate { + return &certv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: cr.Name + "-agent-proxy", + Namespace: cr.InstallNamespace, + }, + Spec: certv1.CertificateSpec{ + CommonName: fmt.Sprintf("%s-agent.%s.svc", cr.Name, cr.InstallNamespace), + DNSNames: []string{ + cr.Name + "-agent", + fmt.Sprintf("%s-agent.%s.svc", cr.Name, cr.InstallNamespace), + fmt.Sprintf("%s-agent.%s.svc.cluster.local", cr.Name, cr.InstallNamespace), + }, + SecretName: cr.Name + "-agent-tls", + IssuerRef: certMeta.ObjectReference{ + Name: cr.Name + "-ca", + }, + Usages: append(certv1.DefaultKeyUsages(), + certv1.UsageServerAuth, + ), + }, + } +} diff --git a/internal/controllers/common/resource_definitions/resource_definitions.go b/internal/controllers/common/resource_definitions/resource_definitions.go index 3b4d337e..6bd44631 100644 --- a/internal/controllers/common/resource_definitions/resource_definitions.go +++ b/internal/controllers/common/resource_definitions/resource_definitions.go @@ -44,6 +44,7 @@ type ImageTags struct { ReportsImageTag string StorageImageTag string DatabaseImageTag string + AgentProxyImageTag string } type ServiceSpecs struct { @@ -61,6 +62,8 @@ type TLSConfig struct { CryostatSecret string // Name of the TLS secret for Reports Generator ReportsSecret string + // Name of the TLS secret for the agent proxy + AgentProxySecret string // Name of the secret containing the password for the keystore in CryostatSecret KeystorePassSecret string // PEM-encoded X.509 certificate for the Cryostat CA @@ -82,6 +85,8 @@ const ( defaultStorageMemoryRequest string = "256Mi" defaultReportCpuRequest string = "500m" defaultReportMemoryRequest string = "512Mi" + defaultAgentProxyCpuRequest string = "25m" + defaultAgentProxyMemoryRequest string = "64Mi" OAuth2ConfigFileName string = "alpha_config.json" OAuth2ConfigFilePath string = "/etc/oauth2_proxy/alpha_config" ) @@ -298,6 +303,8 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima } if tls != nil { + containers = append(containers, newAgentProxyContainer(cr, imageTags.AgentProxyImageTag, tls)) + volSources = append(volSources, corev1.VolumeProjection{ // Add Cryostat self-signed CA Secret: &corev1.SecretProjection{ @@ -323,6 +330,36 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima }, }, }, + corev1.Volume{ + Name: "agent-proxy-tls-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tls.AgentProxySecret, + }, + }, + }, + corev1.Volume{ + Name: "agent-proxy-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cr.Name + "-agent-proxy", + }, + Items: []corev1.KeyToPath{ + { + Key: constants.AgentProxyConfigFileName, + Path: constants.AgentProxyConfigFileName, + Mode: &readOnlyMode, + }, + { + Key: constants.AgentProxyDHFileName, + Path: constants.AgentProxyDHFileName, + Mode: &readOnlyMode, + }, + }, + }, + }, + }, corev1.Volume{ Name: "keystore", VolumeSource: corev1.VolumeSource{ @@ -1161,7 +1198,7 @@ func NewCoreContainer(cr *model.CryostatInstance, specs *ServiceSpecs, imageTag probeHandler := corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Port: intstr.IntOrString{IntVal: constants.CryostatHTTPContainerPort}, - Path: "/health/liveness", + Path: constants.CryostatHealthPath, Scheme: corev1.URISchemeHTTP, }, } @@ -1522,6 +1559,70 @@ func NewJfrDatasourceContainer(cr *model.CryostatInstance, imageTag string) core } } +func newAgentProxyContainer(cr *model.CryostatInstance, imageTag string, tls *TLSConfig) corev1.Container { + var securityContext *corev1.SecurityContext + if cr.Spec.SecurityOptions != nil && cr.Spec.SecurityOptions.AgentProxySecurityContext != nil { + securityContext = cr.Spec.SecurityOptions.AgentProxySecurityContext + } else { + privEscalation := false + securityContext = &corev1.SecurityContext{ + AllowPrivilegeEscalation: &privEscalation, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{constants.CapabilityAll}, + }, + } + } + + healthCheckPort := int32(8081) + return corev1.Container{ + Name: cr.Name + "-agent-proxy", + Image: imageTag, + ImagePullPolicy: getPullPolicy(imageTag), + Ports: []corev1.ContainerPort{ + { + ContainerPort: constants.AgentProxyContainerPort, + }, + }, + Command: []string{ + "nginx", "-c", + fmt.Sprintf("%s/%s", constants.AgentProxyConfigFilePath, constants.AgentProxyConfigFileName), + "-g", "daemon off;"}, + // Can't use HTTP probe since the port is not exposed over the network + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt32(healthCheckPort), + Scheme: corev1.URISchemeHTTP, + }, + }, + }, + SecurityContext: securityContext, + Resources: *newAgentProxyContainerResource(cr), + VolumeMounts: []corev1.VolumeMount{ + { + Name: "agent-proxy-tls-secret", + MountPath: "/var/run/secrets/operator.cryostat.io/" + tls.AgentProxySecret, + ReadOnly: true, + }, + { + Name: "agent-proxy-config", + MountPath: constants.AgentProxyConfigFilePath, + ReadOnly: true, + }, + }, + } +} + +func newAgentProxyContainerResource(cr *model.CryostatInstance) *corev1.ResourceRequirements { + resources := &corev1.ResourceRequirements{} + if cr.Spec.Resources != nil { + resources = cr.Spec.Resources.AgentProxyResources.DeepCopy() + } + populateResourceRequest(resources, defaultJfrDatasourceCpuRequest, defaultJfrDatasourceMemoryRequest) + return resources +} + func getPort(url *url.URL) string { // Return port if already defined in URL port := url.Port() diff --git a/internal/controllers/configmaps.go b/internal/controllers/configmaps.go index adff1187..4d7f7acf 100644 --- a/internal/controllers/configmaps.go +++ b/internal/controllers/configmaps.go @@ -15,9 +15,11 @@ package controllers import ( + "bytes" "context" "encoding/json" "fmt" + "text/template" resources "github.com/cryostatio/cryostat-operator/internal/controllers/common/resource_definitions" "github.com/cryostatio/cryostat-operator/internal/controllers/constants" @@ -35,7 +37,9 @@ func (r *Reconciler) reconcileLockConfigMap(ctx context.Context, cr *model.Cryos Namespace: cr.InstallNamespace, }, } - return r.createOrUpdateConfigMap(ctx, cm, cr.Object) + return r.createOrUpdateConfigMap(ctx, cm, cr.Object, func() error { + return nil + }) } type oauth2ProxyAlphaConfig struct { @@ -82,76 +86,233 @@ type alphaConfigUpstream struct { } func (r *Reconciler) reconcileOAuth2ProxyConfig(ctx context.Context, cr *model.CryostatInstance, tls *resources.TLSConfig) error { - bindHost := "0.0.0.0" - immutable := true - cfg := &oauth2ProxyAlphaConfig{ - Server: alphaConfigServer{}, - UpstreamConfig: alphaConfigUpstreamConfig{ProxyRawPath: true, Upstreams: []alphaConfigUpstream{ - { - Id: "cryostat", - Path: "/", - Uri: fmt.Sprintf("http://localhost:%d", constants.CryostatHTTPContainerPort), - }, - { - Id: "grafana", - Path: "/grafana/", - Uri: fmt.Sprintf("http://localhost:%d", constants.GrafanaContainerPort), - }, - { - Id: "storage", - Path: "^/storage/(.*)$", - RewriteTarget: "/$1", - Uri: fmt.Sprintf("http://localhost:%d", constants.StoragePort), - PassHostHeader: false, - ProxyWebSockets: false, - }, - }}, - Providers: []alphaConfigProvider{{Id: "dummy", Name: "Unused - Sign In Below", ClientId: "CLIENT_ID", ClientSecret: "CLIENT_SECRET", Provider: "google"}}, + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cr.Name + "-oauth2-proxy-cfg", + Namespace: cr.InstallNamespace, + }, } - if tls != nil { - cfg.Server.SecureBindAddress = fmt.Sprintf("https://%s:%d", bindHost, constants.AuthProxyHttpContainerPort) - cfg.Server.TLS = proxyTLS{ - Key: tlsSecretSource{ - FromFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.CryostatSecret, corev1.TLSPrivateKeyKey), - }, - Cert: tlsSecretSource{ - FromFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.CryostatSecret, corev1.TLSCertKey), - }, + if r.IsOpenShift { + return r.deleteConfigMap(ctx, cm) + } + + return r.createOrUpdateConfigMap(ctx, cm, cr.Object, func() error { + bindHost := "0.0.0.0" + immutable := true + cfg := &oauth2ProxyAlphaConfig{ + Server: alphaConfigServer{}, + UpstreamConfig: alphaConfigUpstreamConfig{ProxyRawPath: true, Upstreams: []alphaConfigUpstream{ + { + Id: "cryostat", + Path: "/", + Uri: fmt.Sprintf("http://localhost:%d", constants.CryostatHTTPContainerPort), + }, + { + Id: "grafana", + Path: "/grafana/", + Uri: fmt.Sprintf("http://localhost:%d", constants.GrafanaContainerPort), + }, + { + Id: "storage", + Path: "^/storage/(.*)$", + RewriteTarget: "/$1", + Uri: fmt.Sprintf("http://localhost:%d", constants.StoragePort), + PassHostHeader: false, + ProxyWebSockets: false, + }, + }}, + Providers: []alphaConfigProvider{{Id: "dummy", Name: "Unused - Sign In Below", ClientId: "CLIENT_ID", ClientSecret: "CLIENT_SECRET", Provider: "google"}}, + } + + if tls != nil { + cfg.Server.SecureBindAddress = fmt.Sprintf("https://%s:%d", bindHost, constants.AuthProxyHttpContainerPort) + cfg.Server.TLS = proxyTLS{ + Key: tlsSecretSource{ + FromFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.CryostatSecret, corev1.TLSPrivateKeyKey), + }, + Cert: tlsSecretSource{ + FromFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.CryostatSecret, corev1.TLSCertKey), + }, + } + } else { + cfg.Server.BindAddress = fmt.Sprintf("http://%s:%d", bindHost, constants.AuthProxyHttpContainerPort) + } + + data := make(map[string]string) + json, err := json.Marshal(cfg) + if err != nil { + return err + } + data[resources.OAuth2ConfigFileName] = string(json) + + cm.Immutable = &immutable + cm.Data = data + return nil + }) +} + +type nginxConfParams struct { + ServerName string + TLSCertFile string + TLSKeyFile string + CACertFile string + DHParamFile string + ContainerPort int32 + CryostatPort int32 + CryostatHealthPath string + AllowedPaths []string +} + +// Reference: https://ssl-config.mozilla.org +var nginxConfTemplate = template.Must(template.New("").Parse(` +worker_processes auto; +error_log stderr notice; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /dev/stdout main; + + sendfile on; + tcp_nopush on; + keepalive_timeout 65; + types_hash_max_size 4096; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + server { + listen {{ .ContainerPort }} ssl; + listen [::]:{{ .ContainerPort }} ssl; + server_name {{ .ServerName }}; + + ssl_certificate {{ .TLSCertFile }}; + ssl_certificate_key {{ .TLSKeyFile }}; + + ssl_session_timeout 5m; + ssl_session_cache shared:SSL:20m; + ssl_session_tickets off; + + ssl_dhparam {{ .DHParamFile }}; + + # intermediate configuration + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; + ssl_prefer_server_ciphers off; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + ssl_trusted_certificate {{ .CACertFile }}; + + # Client certificate authentication + ssl_client_certificate {{ .CACertFile }}; + ssl_verify_client on; + + {{ range .AllowedPaths -}} + location {{ . }} { + proxy_pass http://127.0.0.1:{{ $.CryostatPort }}$request_uri; + } + + {{ end -}} + + location / { + return 404; } - } else { - cfg.Server.BindAddress = fmt.Sprintf("http://%s:%d", bindHost, constants.AuthProxyHttpContainerPort) } - data := make(map[string]string) - json, err := json.Marshal(cfg) - if err != nil { - return err + # Heatlh Check + server { + listen 8081; + + location = /healthz { + return 200; + } + + location / { + return 404; + } } - data[resources.OAuth2ConfigFileName] = string(json) +}`)) // TODO can we proxy health check and include all TLS config without client auth? + +// From https://ssl-config.mozilla.org/ffdhe2048.txt +const dhParams = `-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz ++8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a +87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 +YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi +7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD +ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== +-----END DH PARAMETERS-----` + +func (r *Reconciler) reconcileAgentProxyConfig(ctx context.Context, cr *model.CryostatInstance, tls *resources.TLSConfig) error { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: cr.Name + "-oauth2-proxy-cfg", + Name: cr.Name + "-agent-proxy", Namespace: cr.InstallNamespace, }, - Immutable: &immutable, - Data: data, } - if r.IsOpenShift { + if tls == nil { return r.deleteConfigMap(ctx, cm) - } else { - return r.createOrUpdateConfigMap(ctx, cm, cr.Object) } + + buf := &bytes.Buffer{} + params := &nginxConfParams{ + ServerName: fmt.Sprintf("%s-agent.%s.svc", cr.Name, cr.InstallNamespace), + TLSCertFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSCertKey), + TLSKeyFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSPrivateKeyKey), + CACertFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, constants.CAKey), + DHParamFile: fmt.Sprintf("%s/%s", constants.AgentProxyConfigFilePath, constants.AgentProxyDHFileName), + ContainerPort: constants.AgentProxyContainerPort, + CryostatPort: constants.CryostatHTTPContainerPort, + CryostatHealthPath: constants.CryostatHealthPath, + AllowedPaths: []string{ + "/api/v2.2/discovery/", + "/api/v2.2/credentials/", + "/api/beta/recordings/", + }, + } + err := nginxConfTemplate.Execute(buf, params) + if err != nil { + return err + } + + return r.createOrUpdateConfigMap(ctx, cm, cr.Object, func() error { + immutable := true // TODO breaks update + cm.Immutable = &immutable + + cm.Data = map[string]string{ + constants.AgentProxyConfigFileName: buf.String(), + constants.AgentProxyDHFileName: dhParams, + } + return nil + }) } -func (r *Reconciler) createOrUpdateConfigMap(ctx context.Context, cm *corev1.ConfigMap, owner metav1.Object) error { +func (r *Reconciler) createOrUpdateConfigMap(ctx context.Context, cm *corev1.ConfigMap, owner metav1.Object, + delegate controllerutil.MutateFn) error { op, err := controllerutil.CreateOrUpdate(ctx, r.Client, cm, func() error { // Set the Cryostat CR as controller if err := controllerutil.SetControllerReference(owner, cm, r.Scheme); err != nil { return err } - return nil + return delegate() }) if err != nil { return err diff --git a/internal/controllers/const_generated.go b/internal/controllers/const_generated.go index c4822c36..8ab5bced 100644 --- a/internal/controllers/const_generated.go +++ b/internal/controllers/const_generated.go @@ -30,3 +30,6 @@ const DefaultStorageImageTag = "quay.io/cryostat/cryostat-storage:latest" // Default image tag for the Database image const DefaultDatabaseImageTag = "quay.io/cryostat/cryostat-db:latest" + +// Default image tag for the agent proxy image +const DefaultAgentProxyImageTag = "registry.access.redhat.com/ubi8/nginx-124:latest" diff --git a/internal/controllers/constants/constants.go b/internal/controllers/constants/constants.go index 67872189..4772bd34 100644 --- a/internal/controllers/constants/constants.go +++ b/internal/controllers/constants/constants.go @@ -27,6 +27,8 @@ const ( ReportsContainerPort int32 = 10000 StoragePort int32 = 8333 DatabasePort int32 = 5432 + AgentProxyContainerPort int32 = 8443 + AgentProxyHealthPort int32 = 8081 LoopbackAddress string = "127.0.0.1" OperatorNamePrefix string = "cryostat-operator-" OperatorDeploymentName string = "cryostat-operator-controller" @@ -42,6 +44,12 @@ const ( // DatabaseSecretEncryptionKey indexes the database encryption key within the Cryostat database Secret DatabaseSecretEncryptionKey = "ENCRYPTION_KEY" + AgentProxyConfigFilePath string = "/etc/nginx-cryostat" + AgentProxyConfigFileName string = "nginx.conf" + AgentProxyDHFileName string = "dhparam.pem" + + CryostatHealthPath string = "/health/liveness" + targetNamespaceCRLabelPrefix = "operator.cryostat.io/" TargetNamespaceCRNameLabel = targetNamespaceCRLabelPrefix + "name" TargetNamespaceCRNamespaceLabel = targetNamespaceCRLabelPrefix + "namespace" diff --git a/internal/controllers/reconciler.go b/internal/controllers/reconciler.go index 18759da4..4474d4f6 100644 --- a/internal/controllers/reconciler.go +++ b/internal/controllers/reconciler.go @@ -108,6 +108,9 @@ const storageImageTagEnv = "RELATED_IMAGE_STORAGE" // Environment variable to override the cryostat-database image const databaseImageTagEnv = "RELATED_IMAGE_DATABASE" +// Environment variable to override the agent proxy image +const agentProxyImageTagEnv = "RELATED_IMAGE_AGENT_PROXY" + // Regular expression for the start of a GID range in the OpenShift // supplemental groups SCC annotation var supGroupRegexp = regexp.MustCompile(`^\d+`) @@ -254,6 +257,10 @@ func (r *Reconciler) reconcileCryostat(ctx context.Context, cr *model.CryostatIn if err != nil { return reconcile.Result{}, err } + err = r.reconcileAgentProxyConfig(ctx, cr, tlsConfig) + if err != nil { + return reconcile.Result{}, err + } serviceSpecs := &resources.ServiceSpecs{ InsightsURL: r.InsightsProxy, @@ -262,6 +269,10 @@ func (r *Reconciler) reconcileCryostat(ctx context.Context, cr *model.CryostatIn if err != nil { return requeueIfIngressNotReady(reqLogger, err) } + err = r.reconcileAgentService(ctx, cr, tlsConfig, serviceSpecs) + if err != nil { + return reconcile.Result{}, err + } imageTags := r.getImageTags() fsGroup, err := r.getFSGroup(ctx, cr.InstallNamespace) @@ -392,6 +403,7 @@ func (r *Reconciler) getImageTags() *resources.ImageTags { ReportsImageTag: r.getEnvOrDefault(reportsImageTagEnv, DefaultReportsImageTag), StorageImageTag: r.getEnvOrDefault(storageImageTagEnv, DefaultStorageImageTag), DatabaseImageTag: r.getEnvOrDefault(databaseImageTagEnv, DefaultDatabaseImageTag), + AgentProxyImageTag: r.getEnvOrDefault(agentProxyImageTagEnv, DefaultAgentProxyImageTag), } } diff --git a/internal/controllers/services.go b/internal/controllers/services.go index db099e1f..9b8a1ad5 100644 --- a/internal/controllers/services.go +++ b/internal/controllers/services.go @@ -111,6 +111,46 @@ func (r *Reconciler) reconcileReportsService(ctx context.Context, cr *model.Cryo return nil } +func (r *Reconciler) reconcileAgentService(ctx context.Context, cr *model.CryostatInstance, + tls *resource_definitions.TLSConfig, specs *resource_definitions.ServiceSpecs) error { + config := configureAgentService(cr) + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: cr.Name + "-agent", + Namespace: cr.InstallNamespace, + }, + } + + if tls == nil { + // Delete service if it exists + return r.deleteService(ctx, svc) + } + err := r.createOrUpdateService(ctx, svc, cr.Object, &config.ServiceConfig, func() error { + svc.Spec.Selector = map[string]string{ + "app": cr.Name, + "component": "cryostat", + } + svc.Spec.Ports = []corev1.ServicePort{ + { + Name: "http", + Port: *config.HTTPPort, + TargetPort: intstr.IntOrString{IntVal: constants.AgentProxyContainerPort}, + }, + } + return nil + }) + if err != nil { + return err + } + + // Set reports URL for deployment to use + specs.ReportsURL = &url.URL{ + Scheme: "https", + Host: svc.Name + ":" + strconv.Itoa(int(svc.Spec.Ports[0].Port)), // TODO use getHTTPPort? + } + return nil +} + func configureCoreService(cr *model.CryostatInstance) *operatorv1beta2.CoreServiceConfig { // Check CR for config var config *operatorv1beta2.CoreServiceConfig @@ -153,6 +193,27 @@ func configureReportsService(cr *model.CryostatInstance) *operatorv1beta2.Report return config } +func configureAgentService(cr *model.CryostatInstance) *operatorv1beta2.AgentServiceConfig { + // Check CR for config + var config *operatorv1beta2.AgentServiceConfig + if cr.Spec.ServiceOptions == nil || cr.Spec.ServiceOptions.AgentConfig == nil { + config = &operatorv1beta2.AgentServiceConfig{} + } else { + config = cr.Spec.ServiceOptions.AgentConfig.DeepCopy() + } + + // Apply common service defaults + configureService(&config.ServiceConfig, cr.Name, "cryostat") + + // Apply default HTTP port if not provided + if config.HTTPPort == nil { + httpPort := constants.AgentProxyContainerPort + config.HTTPPort = &httpPort + } + + return config +} + func configureService(config *operatorv1beta2.ServiceConfig, appLabel string, componentLabel string) { if config.ServiceType == nil { svcType := corev1.ServiceTypeClusterIP diff --git a/internal/tools/const_generator.go b/internal/tools/const_generator.go index dc2bb4f9..1c60c0ae 100644 --- a/internal/tools/const_generator.go +++ b/internal/tools/const_generator.go @@ -32,6 +32,7 @@ const grafanaImageEnv = "GRAFANA_IMG" const reportsImageEnv = "REPORTS_IMG" const storageImageEnv = "STORAGE_IMG" const databaseImageEnv = "DATABASE_IMG" +const agentProxyImageEnv = "AGENT_PROXY_IMG" // This program generates a const_generated.go file containing image tag // constants for each container image deployed by the operator, along with @@ -39,27 +40,29 @@ const databaseImageEnv = "DATABASE_IMG" func main() { // Fill in image tags struct from the environment variables consts := struct { - AppName string - OperatorVersion string - OAuth2ProxyImageTag string + AppName string + OperatorVersion string + OAuth2ProxyImageTag string OpenShiftOAuthProxyImageTag string - CoreImageTag string - DatasourceImageTag string - GrafanaImageTag string - ReportsImageTag string - StorageImageTag string - DatabaseImageTag string + CoreImageTag string + DatasourceImageTag string + GrafanaImageTag string + ReportsImageTag string + StorageImageTag string + DatabaseImageTag string + AgentProxyImageTag string }{ - AppName: getEnvVar(appNameEnv), - OperatorVersion: getEnvVar(operatorVersionEnv), - OAuth2ProxyImageTag: getEnvVar(oauth2ProxyImageEnv), + AppName: getEnvVar(appNameEnv), + OperatorVersion: getEnvVar(operatorVersionEnv), + OAuth2ProxyImageTag: getEnvVar(oauth2ProxyImageEnv), OpenShiftOAuthProxyImageTag: getEnvVar(openshiftOauthProxyImageEnv), - CoreImageTag: getEnvVar(coreImageEnv), - DatasourceImageTag: getEnvVar(datasourceImageEnv), - GrafanaImageTag: getEnvVar(grafanaImageEnv), - ReportsImageTag: getEnvVar(reportsImageEnv), - StorageImageTag: getEnvVar(storageImageEnv), - DatabaseImageTag: getEnvVar(databaseImageEnv), + CoreImageTag: getEnvVar(coreImageEnv), + DatasourceImageTag: getEnvVar(datasourceImageEnv), + GrafanaImageTag: getEnvVar(grafanaImageEnv), + ReportsImageTag: getEnvVar(reportsImageEnv), + StorageImageTag: getEnvVar(storageImageEnv), + DatabaseImageTag: getEnvVar(databaseImageEnv), + AgentProxyImageTag: getEnvVar(agentProxyImageEnv), } // Create the source file to generate @@ -115,4 +118,7 @@ const DefaultStorageImageTag = "{{ .StorageImageTag }}" // Default image tag for the Database image const DefaultDatabaseImageTag = "{{ .DatabaseImageTag }}" + +// Default image tag for the agent proxy image +const DefaultAgentProxyImageTag = "{{ .AgentProxyImageTag }}" `)) From bf860bcda1b8bc01a79433f7788d3d9bbc4a29b3 Mon Sep 17 00:00:00 2001 From: Elliott Baron Date: Fri, 20 Sep 2024 14:00:36 -0400 Subject: [PATCH 2/6] Add/fix test cases --- api/v1beta1/cryostat_conversion_test.go | 25 +- ...yostat-operator.clusterserviceversion.yaml | 2 +- .../resource_definitions.go | 26 +- internal/controllers/configmaps.go | 37 +- internal/controllers/constants/constants.go | 4 +- internal/controllers/reconciler_test.go | 108 +++- internal/controllers/services.go | 12 +- internal/test/clients.go | 2 +- internal/test/conversion.go | 102 +++- internal/test/reconciler.go | 4 + internal/test/resources.go | 463 +++++++++++++++--- 11 files changed, 627 insertions(+), 158 deletions(-) diff --git a/api/v1beta1/cryostat_conversion_test.go b/api/v1beta1/cryostat_conversion_test.go index a0058def..6d28ceb7 100644 --- a/api/v1beta1/cryostat_conversion_test.go +++ b/api/v1beta1/cryostat_conversion_test.go @@ -80,17 +80,32 @@ func tableEntriesTo() []TableEntry { return append(tableEntries(), Entry("WS connections", (*test.TestResources).NewCryostatWithWsConnectionsSpecV1Beta1, (*test.TestResources).NewCryostat), - Entry("command config", (*test.TestResources).NewCryostatWithCommandConfigV1Beta1, + Entry("command ingress", (*test.TestResources).NewCryostatWithCommandConfigV1Beta1, + (*test.TestResources).NewCryostatWithIngress), + Entry("grafana ingress", (*test.TestResources).NewCryostatWithGrafanaConfigV1Beta1, (*test.TestResources).NewCryostatWithIngress), Entry("minimal mode", (*test.TestResources).NewCryostatWithMinimalModeV1Beta1, (*test.TestResources).NewCryostat), Entry("core JMX port", (*test.TestResources).NewCryostatWithCoreSvcJMXPortV1Beta1, (*test.TestResources).NewCryostatWithCoreSvc), + Entry("resources", (*test.TestResources).NewCryostatWithResourcesV1Beta1, + (*test.TestResources).NewCryostatWithResourcesToV1Beta2), + Entry("low resource limit", (*test.TestResources).NewCryostatWithLowResourceLimitV1Beta1, + (*test.TestResources).NewCryostatWithLowResourceLimitToV1Beta2), + Entry("security", (*test.TestResources).NewCryostatWithSecurityOptionsV1Beta1, + (*test.TestResources).NewCryostatWithSecurityOptionsToV1Beta2), ) } func tableEntriesFrom() []TableEntry { - return tableEntries() + return append(tableEntries(), + Entry("resources", (*test.TestResources).NewCryostatWithResourcesV1Beta1, + (*test.TestResources).NewCryostatWithResources), + Entry("low resource limit", (*test.TestResources).NewCryostatWithLowResourceLimitV1Beta1, + (*test.TestResources).NewCryostatWithLowResourceLimit), + Entry("security", (*test.TestResources).NewCryostatWithSecurityOptionsV1Beta1, + (*test.TestResources).NewCryostatWithSecurityOptions), + ) } func tableEntries() []TableEntry { @@ -133,10 +148,6 @@ func tableEntries() []TableEntry { (*test.TestResources).NewCryostatCertManagerDisabled), Entry("cert-manager undefined", (*test.TestResources).NewCryostatCertManagerUndefinedV1Beta1, (*test.TestResources).NewCryostatCertManagerUndefined), - Entry("resources", (*test.TestResources).NewCryostatWithResourcesV1Beta1, - (*test.TestResources).NewCryostatWithResources), - Entry("low resource limit", (*test.TestResources).NewCryostatWithLowResourceLimitV1Beta1, - (*test.TestResources).NewCryostatWithLowResourceLimit), Entry("built-in discovery disabled", (*test.TestResources).NewCryostatWithBuiltInDiscoveryDisabledV1Beta1, (*test.TestResources).NewCryostatWithBuiltInDiscoveryDisabled), Entry("discovery port custom config", (*test.TestResources).NewCryostatWithDiscoveryPortConfigV1Beta1, @@ -145,8 +156,6 @@ func tableEntries() []TableEntry { (*test.TestResources).NewCryostatWithBuiltInPortConfigDisabled), Entry("JMX cache options", (*test.TestResources).NewCryostatWithJmxCacheOptionsSpecV1Beta1, (*test.TestResources).NewCryostatWithJmxCacheOptionsSpec), - Entry("security", (*test.TestResources).NewCryostatWithSecurityOptionsV1Beta1, - (*test.TestResources).NewCryostatWithSecurityOptions), Entry("reports security", (*test.TestResources).NewCryostatWithReportSecurityOptionsV1Beta1, (*test.TestResources).NewCryostatWithReportSecurityOptions), Entry("database secret", (*test.TestResources).NewCryostatWithDatabaseSecretProvidedV1Beta1, diff --git a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml index bc9fc802..730c6ead 100644 --- a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml +++ b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml @@ -30,7 +30,7 @@ metadata: capabilities: Seamless Upgrades categories: Monitoring, Developer Tools containerImage: quay.io/cryostat/cryostat-operator:4.0.0-dev - createdAt: "2024-09-19T13:48:41Z" + createdAt: "2024-09-20T18:00:33Z" description: JVM monitoring and profiling tool operatorframework.io/initialization-resource: |- { diff --git a/internal/controllers/common/resource_definitions/resource_definitions.go b/internal/controllers/common/resource_definitions/resource_definitions.go index 6bd44631..672b24cd 100644 --- a/internal/controllers/common/resource_definitions/resource_definitions.go +++ b/internal/controllers/common/resource_definitions/resource_definitions.go @@ -326,7 +326,8 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima Name: "auth-proxy-tls-secret", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: tls.CryostatSecret, + SecretName: tls.CryostatSecret, + DefaultMode: &readOnlyMode, }, }, }, @@ -334,7 +335,8 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima Name: "agent-proxy-tls-secret", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: tls.AgentProxySecret, + SecretName: tls.AgentProxySecret, + DefaultMode: &readOnlyMode, }, }, }, @@ -345,18 +347,7 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima LocalObjectReference: corev1.LocalObjectReference{ Name: cr.Name + "-agent-proxy", }, - Items: []corev1.KeyToPath{ - { - Key: constants.AgentProxyConfigFileName, - Path: constants.AgentProxyConfigFileName, - Mode: &readOnlyMode, - }, - { - Key: constants.AgentProxyDHFileName, - Path: constants.AgentProxyDHFileName, - Mode: &readOnlyMode, - }, - }, + DefaultMode: &readOnlyMode, }, }, }, @@ -1198,7 +1189,7 @@ func NewCoreContainer(cr *model.CryostatInstance, specs *ServiceSpecs, imageTag probeHandler := corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Port: intstr.IntOrString{IntVal: constants.CryostatHTTPContainerPort}, - Path: constants.CryostatHealthPath, + Path: "/health/liveness", Scheme: corev1.URISchemeHTTP, }, } @@ -1582,6 +1573,9 @@ func newAgentProxyContainer(cr *model.CryostatInstance, imageTag string, tls *TL { ContainerPort: constants.AgentProxyContainerPort, }, + { + ContainerPort: healthCheckPort, + }, }, Command: []string{ "nginx", "-c", @@ -1619,7 +1613,7 @@ func newAgentProxyContainerResource(cr *model.CryostatInstance) *corev1.Resource if cr.Spec.Resources != nil { resources = cr.Spec.Resources.AgentProxyResources.DeepCopy() } - populateResourceRequest(resources, defaultJfrDatasourceCpuRequest, defaultJfrDatasourceMemoryRequest) + populateResourceRequest(resources, defaultAgentProxyCpuRequest, defaultAgentProxyMemoryRequest) return resources } diff --git a/internal/controllers/configmaps.go b/internal/controllers/configmaps.go index 4d7f7acf..7f3ba423 100644 --- a/internal/controllers/configmaps.go +++ b/internal/controllers/configmaps.go @@ -153,15 +153,14 @@ func (r *Reconciler) reconcileOAuth2ProxyConfig(ctx context.Context, cr *model.C } type nginxConfParams struct { - ServerName string - TLSCertFile string - TLSKeyFile string - CACertFile string - DHParamFile string - ContainerPort int32 - CryostatPort int32 - CryostatHealthPath string - AllowedPaths []string + ServerName string + TLSCertFile string + TLSKeyFile string + CACertFile string + DHParamFile string + ContainerPort int32 + CryostatPort int32 + AllowedPaths []string } // Reference: https://ssl-config.mozilla.org @@ -248,7 +247,7 @@ http { return 404; } } -}`)) // TODO can we proxy health check and include all TLS config without client auth? +}`)) // From https://ssl-config.mozilla.org/ffdhe2048.txt const dhParams = `-----BEGIN DH PARAMETERS----- @@ -268,24 +267,24 @@ func (r *Reconciler) reconcileAgentProxyConfig(ctx context.Context, cr *model.Cr }, } - if tls == nil { + if tls == nil { // TODO make this work without TLS return r.deleteConfigMap(ctx, cm) } buf := &bytes.Buffer{} params := &nginxConfParams{ - ServerName: fmt.Sprintf("%s-agent.%s.svc", cr.Name, cr.InstallNamespace), - TLSCertFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSCertKey), - TLSKeyFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSPrivateKeyKey), - CACertFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, constants.CAKey), - DHParamFile: fmt.Sprintf("%s/%s", constants.AgentProxyConfigFilePath, constants.AgentProxyDHFileName), - ContainerPort: constants.AgentProxyContainerPort, - CryostatPort: constants.CryostatHTTPContainerPort, - CryostatHealthPath: constants.CryostatHealthPath, + ServerName: fmt.Sprintf("%s-agent.%s.svc", cr.Name, cr.InstallNamespace), + TLSCertFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSCertKey), + TLSKeyFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSPrivateKeyKey), + CACertFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, constants.CAKey), + DHParamFile: fmt.Sprintf("%s/%s", constants.AgentProxyConfigFilePath, constants.AgentProxyDHFileName), + ContainerPort: constants.AgentProxyContainerPort, + CryostatPort: constants.CryostatHTTPContainerPort, AllowedPaths: []string{ "/api/v2.2/discovery/", "/api/v2.2/credentials/", "/api/beta/recordings/", + "/health/", }, } err := nginxConfTemplate.Execute(buf, params) diff --git a/internal/controllers/constants/constants.go b/internal/controllers/constants/constants.go index 4772bd34..37c2f723 100644 --- a/internal/controllers/constants/constants.go +++ b/internal/controllers/constants/constants.go @@ -46,9 +46,7 @@ const ( AgentProxyConfigFilePath string = "/etc/nginx-cryostat" AgentProxyConfigFileName string = "nginx.conf" - AgentProxyDHFileName string = "dhparam.pem" - - CryostatHealthPath string = "/health/liveness" + AgentProxyDHFileName string = "dhparam.pem" // TODO non-exported targetNamespaceCRLabelPrefix = "operator.cryostat.io/" TargetNamespaceCRNameLabel = targetNamespaceCRLabelPrefix + "name" diff --git a/internal/controllers/reconciler_test.go b/internal/controllers/reconciler_test.go index c18262a4..f4dc86d4 100644 --- a/internal/controllers/reconciler_test.go +++ b/internal/controllers/reconciler_test.go @@ -158,6 +158,8 @@ func resourceChecks() []resourceCheck { {(*cryostatTestInput).expectCoreService, "core service"}, {(*cryostatTestInput).expectMainDeployment, "main deployment"}, {(*cryostatTestInput).expectLockConfigMap, "lock config map"}, + {(*cryostatTestInput).expectAgentProxyConfigMap, "agent proxy config map"}, + {(*cryostatTestInput).expectAgentProxyService, "agent proxy service"}, } } @@ -543,7 +545,7 @@ func (c *controllerTest) commonTests() { It("should configure deployment appropriately", func() { t.expectMainDeployment() t.checkReportsDeployment() - t.checkService(t.Name+"-reports", t.NewReportsService()) + t.checkService(t.NewReportsService()) }) }) Context("with Scheduling options", func() { @@ -562,7 +564,7 @@ func (c *controllerTest) commonTests() { It("should configure deployment appropriately", func() { t.expectMainDeployment() t.checkReportsDeployment() - t.checkService(t.Name+"-reports", t.NewReportsService()) + t.checkService(t.NewReportsService()) }) }) Context("with low limits", func() { @@ -572,7 +574,7 @@ func (c *controllerTest) commonTests() { It("should configure deployment appropriately", func() { t.expectMainDeployment() t.checkReportsDeployment() - t.checkService(t.Name+"-reports", t.NewReportsService()) + t.checkService(t.NewReportsService()) }) }) }) @@ -634,7 +636,7 @@ func (c *controllerTest) commonTests() { It("should configure deployment appropriately", func() { t.expectMainDeployment() t.checkReportsDeployment() - t.checkService(t.Name+"-reports", t.NewReportsService()) + t.checkService(t.NewReportsService()) }) }) Context("Switching from 1 report sidecar to 2", func() { @@ -656,7 +658,7 @@ func (c *controllerTest) commonTests() { It("should configure deployment appropriately", func() { t.expectMainDeployment() t.checkReportsDeployment() - t.checkService(t.Name+"-reports", t.NewReportsService()) + t.checkService(t.NewReportsService()) }) }) Context("Switching from 2 report sidecars to 1", func() { @@ -678,7 +680,7 @@ func (c *controllerTest) commonTests() { It("should configure deployment appropriately", func() { t.expectMainDeployment() t.checkReportsDeployment() - t.checkService(t.Name+"-reports", t.NewReportsService()) + t.checkService(t.NewReportsService()) }) }) Context("Switching from 1 report sidecar to 0", func() { @@ -942,6 +944,7 @@ func (c *controllerTest) commonTests() { databaseImg := "my/database-image:1.0.0-dev" oauth2ProxyImg := "my/auth-proxy:1.0.0-dev" openshiftAuthProxyImg := "my/openshift-auth-proxy:1.0.0-dev" + agentProxyImg := "my/agent-proxy:1.0.0-dev" t.EnvCoreImageTag = &coreImg t.EnvDatasourceImageTag = &datasourceImg t.EnvGrafanaImageTag = &grafanaImg @@ -950,6 +953,7 @@ func (c *controllerTest) commonTests() { t.EnvStorageImageTag = &storageImg t.EnvOAuth2ProxyImageTag = &oauth2ProxyImg t.EnvOpenShiftOAuthProxyImageTag = &openshiftAuthProxyImg + t.EnvAgentProxyImageTag = &agentProxyImg }) It("should create deployment with the expected tags", func() { t.expectMainDeployment() @@ -957,7 +961,7 @@ func (c *controllerTest) commonTests() { }) It("should set ImagePullPolicy to Always", func() { containers := mainDeploy.Spec.Template.Spec.Containers - Expect(containers).To(HaveLen(6)) + Expect(containers).To(HaveLen(7)) for _, container := range containers { Expect(container.ImagePullPolicy).To(Equal(corev1.PullAlways)) } @@ -976,6 +980,7 @@ func (c *controllerTest) commonTests() { databaseImg := "my/database-image:1.0.0" oauth2ProxyImg := "my/authproxy-image:1.0.0" openshiftAuthProxyImg := "my/openshift-authproxy-image:1.0.0" + agentProxyImg := "my/agent-proxy:1.0.0" t.EnvCoreImageTag = &coreImg t.EnvDatasourceImageTag = &datasourceImg t.EnvGrafanaImageTag = &grafanaImg @@ -984,6 +989,7 @@ func (c *controllerTest) commonTests() { t.EnvStorageImageTag = &storageImg t.EnvOAuth2ProxyImageTag = &oauth2ProxyImg t.EnvOpenShiftOAuthProxyImageTag = &openshiftAuthProxyImg + t.EnvAgentProxyImageTag = &agentProxyImg }) JustBeforeEach(func() { t.reconcileCryostatFully() @@ -994,7 +1000,7 @@ func (c *controllerTest) commonTests() { }) It("should set ImagePullPolicy to IfNotPresent", func() { containers := mainDeploy.Spec.Template.Spec.Containers - Expect(containers).To(HaveLen(6)) + Expect(containers).To(HaveLen(7)) for _, container := range containers { fmt.Println(container.Image) Expect(container.ImagePullPolicy).To(Equal(corev1.PullIfNotPresent)) @@ -1014,6 +1020,7 @@ func (c *controllerTest) commonTests() { databaseImg := "my/database-image@sha256:8c23ca5e8c8a343789b8c14558a44a49d35ecd130c18e62edf0d1ad9ce88d37d" oauth2ProxyImage := "my/authproxy-image@sha256:8c23ca5e8c8a343789b8c14558a44a49d35ecd130c18e62edf0d1ad9ce88d37d" openshiftAuthProxyImage := "my/openshift-authproxy-image@sha256:8c23ca5e8c8a343789b8c14558a44a49d35ecd130c18e62edf0d1ad9ce88d37d" + agentProxyImg := "my/agent-proxy@sha256:2da2edd513ce134e1b99ea61e84b794c2dece7bd24b9949cc267a1c29020f26a" t.EnvCoreImageTag = &coreImg t.EnvDatasourceImageTag = &datasourceImg t.EnvGrafanaImageTag = &grafanaImg @@ -1022,6 +1029,7 @@ func (c *controllerTest) commonTests() { t.EnvStorageImageTag = &storageImg t.EnvOAuth2ProxyImageTag = &oauth2ProxyImage t.EnvOpenShiftOAuthProxyImageTag = &openshiftAuthProxyImage + t.EnvAgentProxyImageTag = &agentProxyImg }) It("should create deployment with the expected tags", func() { t.expectMainDeployment() @@ -1029,7 +1037,7 @@ func (c *controllerTest) commonTests() { }) It("should set ImagePullPolicy to IfNotPresent", func() { containers := mainDeploy.Spec.Template.Spec.Containers - Expect(containers).To(HaveLen(6)) + Expect(containers).To(HaveLen(7)) for _, container := range containers { Expect(container.ImagePullPolicy).To(Equal(corev1.PullIfNotPresent)) } @@ -1048,6 +1056,7 @@ func (c *controllerTest) commonTests() { openshiftAuthProxyImg := "my/openshift-authproxy-image:latest" dbImg := "my/db-image:latest" storageImg := "my/storage-image:latest" + agentProxyImg := "my/agent-proxy:latest" t.EnvCoreImageTag = &coreImg t.EnvDatasourceImageTag = &datasourceImg t.EnvGrafanaImageTag = &grafanaImg @@ -1056,6 +1065,7 @@ func (c *controllerTest) commonTests() { t.EnvOpenShiftOAuthProxyImageTag = &openshiftAuthProxyImg t.EnvDatabaseImageTag = &dbImg t.EnvStorageImageTag = &storageImg + t.EnvAgentProxyImageTag = &agentProxyImg }) It("should create deployment with the expected tags", func() { t.expectMainDeployment() @@ -1063,7 +1073,7 @@ func (c *controllerTest) commonTests() { }) It("should set ImagePullPolicy to Always", func() { containers := mainDeploy.Spec.Template.Spec.Containers - Expect(containers).To(HaveLen(6)) + Expect(containers).To(HaveLen(7)) for _, container := range containers { Expect(container.ImagePullPolicy).To(Equal(corev1.PullAlways), "Container %s", container.Image) } @@ -1416,7 +1426,7 @@ func (c *controllerTest) commonTests() { t.objs = append(t.objs, t.NewCryostatWithCoreSvc().Object) }) It("should create the service as described", func() { - t.checkService(t.Name, t.NewCustomizedCoreService()) + t.checkService(t.NewCustomizedCoreService()) }) }) Context("containing reports config", func() { @@ -1425,7 +1435,15 @@ func (c *controllerTest) commonTests() { t.objs = append(t.objs, t.NewCryostatWithReportsSvc().Object) }) It("should create the service as described", func() { - t.checkService(t.Name+"-reports", t.NewCustomizedReportsService()) + t.checkService(t.NewCustomizedReportsService()) + }) + }) + Context("containing agent proxy config", func() { + BeforeEach(func() { + t.objs = append(t.objs, t.NewCryostatWithAgentSvc().Object) + }) + It("should create the service as described", func() { + t.checkService(t.NewCustomizedAgentService()) }) }) Context("and existing services", func() { @@ -1448,7 +1466,7 @@ func (c *controllerTest) commonTests() { cr = t.NewCryostatWithCoreSvc() }) It("should create the service as described", func() { - t.checkService(t.Name, t.NewCustomizedCoreService()) + t.checkService(t.NewCustomizedCoreService()) }) }) Context("containing reports config", func() { @@ -1457,7 +1475,15 @@ func (c *controllerTest) commonTests() { cr = t.NewCryostatWithReportsSvc() }) It("should create the service as described", func() { - t.checkService(t.Name+"-reports", t.NewCustomizedReportsService()) + t.checkService(t.NewCustomizedReportsService()) + }) + }) + Context("containing agent proxy config", func() { + BeforeEach(func() { + cr = t.NewCryostatWithAgentSvc() + }) + It("should create the service as described", func() { + t.checkService(t.NewCustomizedAgentService()) }) }) }) @@ -1779,7 +1805,7 @@ func (c *controllerTest) commonTests() { t.expectMainDeployment() }) - It("should create CA Cert secret in each namespace", func() { + It("should create certificate secrets in each namespace", func() { t.expectCertificates() }) @@ -2039,7 +2065,7 @@ func (c *controllerTest) commonTests() { t.checkReportsDeployment() }) It("should create the reports service", func() { - t.checkService(t.Name+"-reports", t.NewReportsService()) + t.checkService(t.NewReportsService()) }) }) Context("with security options", func() { @@ -2528,7 +2554,7 @@ func (t *cryostatTestInput) expectWaitingForCertificate() { func (t *cryostatTestInput) expectCertificates() { // Check certificates - certs := []*certv1.Certificate{t.NewCryostatCert(), t.NewCACert(), t.NewReportsCert()} + certs := []*certv1.Certificate{t.NewCryostatCert(), t.NewCACert(), t.NewReportsCert(), t.NewAgentProxyCert()} for _, expected := range certs { actual := &certv1.Certificate{} err := t.Client.Get(context.Background(), types.NamespacedName{Name: expected.Name, Namespace: expected.Namespace}, actual) @@ -2701,6 +2727,15 @@ func (t *cryostatTestInput) expectLockConfigMap() { t.checkMetadata(cm, expected) } +func (t *cryostatTestInput) expectAgentProxyConfigMap() { + expected := t.NewAgentProxyConfigMap() + cm := &corev1.ConfigMap{} + err := t.Client.Get(context.Background(), types.NamespacedName{Name: expected.Name, Namespace: expected.Namespace}, cm) + Expect(err).ToNot(HaveOccurred()) + + t.checkMetadata(cm, expected) +} + func (t *cryostatTestInput) expectPVC(expectedPVC *corev1.PersistentVolumeClaim) { pvc := &corev1.PersistentVolumeClaim{} err := t.Client.Get(context.Background(), types.NamespacedName{Name: t.Name, Namespace: t.Namespace}, pvc) @@ -2758,7 +2793,11 @@ func (t *cryostatTestInput) expectStorageSecret() { } func (t *cryostatTestInput) expectCoreService() { - t.checkService(t.Name, t.NewCryostatService()) + t.checkService(t.NewCryostatService()) +} + +func (t *cryostatTestInput) expectAgentProxyService() { + t.checkService(t.NewAgentProxyService()) } func (t *cryostatTestInput) expectStatusApplicationURL() { @@ -2806,9 +2845,9 @@ func (t *cryostatTestInput) expectCryostatFinalizerPresent() { Expect(cr.Object.GetFinalizers()).To(ContainElement("operator.cryostat.io/cryostat.finalizer")) } -func (t *cryostatTestInput) checkService(svcName string, expected *corev1.Service) { +func (t *cryostatTestInput) checkService(expected *corev1.Service) { service := &corev1.Service{} - err := t.Client.Get(context.Background(), types.NamespacedName{Name: svcName, Namespace: t.Namespace}, service) + err := t.Client.Get(context.Background(), types.NamespacedName{Name: expected.Name, Namespace: expected.Namespace}, service) Expect(err).ToNot(HaveOccurred()) t.checkMetadata(service, expected) @@ -3006,6 +3045,13 @@ func (t *cryostatTestInput) checkMainPodTemplate(deployment *appsv1.Deployment, authProxyContainer := template.Spec.Containers[5] t.checkAuthProxyContainer(&authProxyContainer, t.NewAuthProxyContainerResource(cr), t.NewAuthProxySecurityContext(cr), cr.Spec.AuthorizationOptions) + // TODO make this work without TLS + if t.TLS { + // Check that Agent Proxy is configured properly + agentProxyContainer := template.Spec.Containers[6] + t.checkAgentProxyContainer(&agentProxyContainer, t.NewAgentProxyContainerResource(cr), t.NewAgentProxySecurityContext(cr)) + } + // Check that the proper Service Account is set Expect(template.Spec.ServiceAccountName).To(Equal(t.Name)) @@ -3255,6 +3301,28 @@ func (t *cryostatTestInput) checkAuthProxyContainer(container *corev1.Container, test.ExpectResourceRequirements(&container.Resources, resources) } +func (t *cryostatTestInput) checkAgentProxyContainer(container *corev1.Container, resources *corev1.ResourceRequirements, securityContext *corev1.SecurityContext) { + Expect(container.Name).To(Equal(t.Name + "-agent-proxy")) + + imageTag := t.EnvAgentProxyImageTag + defaultPrefix := "registry.access.redhat.com/ubi8/nginx-124:" + if imageTag != nil { + Expect(container.Image).To(Equal(*imageTag)) + } else { + Expect(container.Image).To(HavePrefix(defaultPrefix)) + } + + Expect(container.Ports).To(ConsistOf(t.NewAgentProxyPorts())) + Expect(container.Env).To(ConsistOf(t.NewAgentProxyEnvironmentVariables())) + Expect(container.EnvFrom).To(ConsistOf(t.NewAgentProxyEnvFromSource())) + Expect(container.VolumeMounts).To(ConsistOf(t.NewAgentProxyVolumeMounts())) + Expect(container.LivenessProbe).To(Equal(t.NewAgentProxyLivenessProbe())) + Expect(container.SecurityContext).To(Equal(securityContext)) + Expect(container.Command).To(Equal(t.NewAgentProxyCommand())) + + test.ExpectResourceRequirements(&container.Resources, resources) +} + func (t *cryostatTestInput) checkReportsContainer(container *corev1.Container, resources *corev1.ResourceRequirements, securityContext *corev1.SecurityContext) { Expect(container.Name).To(Equal(t.Name + "-reports")) if t.EnvReportsImageTag == nil { diff --git a/internal/controllers/services.go b/internal/controllers/services.go index 9b8a1ad5..75697c2a 100644 --- a/internal/controllers/services.go +++ b/internal/controllers/services.go @@ -125,7 +125,7 @@ func (r *Reconciler) reconcileAgentService(ctx context.Context, cr *model.Cryost // Delete service if it exists return r.deleteService(ctx, svc) } - err := r.createOrUpdateService(ctx, svc, cr.Object, &config.ServiceConfig, func() error { + return r.createOrUpdateService(ctx, svc, cr.Object, &config.ServiceConfig, func() error { svc.Spec.Selector = map[string]string{ "app": cr.Name, "component": "cryostat", @@ -139,16 +139,6 @@ func (r *Reconciler) reconcileAgentService(ctx context.Context, cr *model.Cryost } return nil }) - if err != nil { - return err - } - - // Set reports URL for deployment to use - specs.ReportsURL = &url.URL{ - Scheme: "https", - Host: svc.Name + ":" + strconv.Itoa(int(svc.Spec.Ports[0].Port)), // TODO use getHTTPPort? - } - return nil } func configureCoreService(cr *model.CryostatInstance) *operatorv1beta2.CoreServiceConfig { diff --git a/internal/test/clients.go b/internal/test/clients.go index 483fd45a..4f6e032b 100644 --- a/internal/test/clients.go +++ b/internal/test/clients.go @@ -115,7 +115,7 @@ func (c *testClient) updateRouteStatus(ctx context.Context, obj runtime.Object) } func (c *testClient) matchesCert(cert *certv1.Certificate) bool { - return c.matchesName(cert, c.NewCryostatCert(), c.NewCACert(), c.NewReportsCert()) || + return c.matchesName(cert, c.NewCryostatCert(), c.NewCACert(), c.NewReportsCert(), c.NewAgentProxyCert()) || c.matchesPrefix(cert, c.GetAgentCertPrefix()) } diff --git a/internal/test/conversion.go b/internal/test/conversion.go index ee42357e..9ac8e3ff 100644 --- a/internal/test/conversion.go +++ b/internal/test/conversion.go @@ -16,10 +16,12 @@ package test import ( operatorv1beta1 "github.com/cryostatio/cryostat-operator/api/v1beta1" + "github.com/cryostatio/cryostat-operator/internal/controllers/model" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) // v1beta1 versions of Cryostat CRs used for testing the conversion webhook @@ -418,6 +420,16 @@ func (r *TestResources) NewCryostatWithResourcesV1Beta1() *operatorv1beta1.Cryos return cr } +// Converted from v1beta1 to v1beta2 +func (r *TestResources) NewCryostatWithResourcesToV1Beta2() *model.CryostatInstance { + cr := r.NewCryostatWithResources() + cr.Spec.Resources.DatabaseResources = corev1.ResourceRequirements{} + cr.Spec.Resources.ObjectStorageResources = corev1.ResourceRequirements{} + cr.Spec.Resources.AuthProxyResources = corev1.ResourceRequirements{} + cr.Spec.Resources.AgentProxyResources = corev1.ResourceRequirements{} + return cr +} + func (r *TestResources) NewCryostatWithLowResourceLimitV1Beta1() *operatorv1beta1.Cryostat { cr := r.NewCryostatV1Beta1() cr.Spec.Resources = &operatorv1beta1.ResourceConfigList{ @@ -443,6 +455,16 @@ func (r *TestResources) NewCryostatWithLowResourceLimitV1Beta1() *operatorv1beta return cr } +// Converted from v1beta1 to v1beta2 +func (r *TestResources) NewCryostatWithLowResourceLimitToV1Beta2() *model.CryostatInstance { + cr := r.NewCryostatWithLowResourceLimit() + cr.Spec.Resources.DatabaseResources = corev1.ResourceRequirements{} + cr.Spec.Resources.ObjectStorageResources = corev1.ResourceRequirements{} + cr.Spec.Resources.AuthProxyResources = corev1.ResourceRequirements{} + cr.Spec.Resources.AgentProxyResources = corev1.ResourceRequirements{} + return cr +} + func (r *TestResources) NewCryostatWithAuthPropertiesV1Beta1() *operatorv1beta1.Cryostat { cr := r.NewCryostatV1Beta1() cr.Spec.AuthProperties = &operatorv1beta1.AuthorizationProperties{ @@ -494,8 +516,35 @@ func (r *TestResources) NewCryostatWithWsConnectionsSpecV1Beta1() *operatorv1bet return cr } +func (r *TestResources) newCommandService() *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.Name + "-command", + Namespace: r.Namespace, + Labels: map[string]string{ + "app": r.Name, + "component": "cryostat", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + "app": r.Name, + "component": "cryostat", + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 10001, + TargetPort: intstr.FromInt(10001), + }, + }, + }, + } +} + func (r *TestResources) NewCryostatWithCommandConfigV1Beta1() *operatorv1beta1.Cryostat { - commandSVC := r.NewCommandService() + commandSVC := r.newCommandService() commandIng := r.newNetworkConfigurationV1Beta1(commandSVC.Name, commandSVC.Spec.Ports[0].Port) commandIng.Annotations["command"] = "annotation" commandIng.Labels["command"] = "label" @@ -509,6 +558,47 @@ func (r *TestResources) NewCryostatWithCommandConfigV1Beta1() *operatorv1beta1.C return cr } +func (r *TestResources) newGrafanaService() *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.Name + "-grafana", + Namespace: r.Namespace, + Labels: map[string]string{ + "app": r.Name, + "component": "cryostat", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + "app": r.Name, + "component": "cryostat", + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 3000, + TargetPort: intstr.FromInt(3000), + }, + }, + }, + } +} + +func (r *TestResources) NewCryostatWithGrafanaConfigV1Beta1() *operatorv1beta1.Cryostat { + grafanaSVC := r.newGrafanaService() + grafanaIng := r.newNetworkConfigurationV1Beta1(grafanaSVC.Name, grafanaSVC.Spec.Ports[0].Port) + grafanaIng.Annotations["command"] = "annotation" + grafanaIng.Labels["command"] = "label" + + cr := r.NewCryostatWithIngressV1Beta1() + cr.Spec.NetworkOptions = &operatorv1beta1.NetworkConfigurationList{ + CoreConfig: cr.Spec.NetworkOptions.CoreConfig, + GrafanaConfig: &grafanaIng, + } + return cr +} + func (r *TestResources) NewCryostatWithReportSubprocessHeapSpecV1Beta1() *operatorv1beta1.Cryostat { cr := r.NewCryostatV1Beta1() if cr.Spec.ReportOptions == nil { @@ -557,6 +647,16 @@ func (r *TestResources) NewCryostatWithSecurityOptionsV1Beta1() *operatorv1beta1 return cr } +// Converted from v1beta1 to v1beta2 +func (r *TestResources) NewCryostatWithSecurityOptionsToV1Beta2() *model.CryostatInstance { + cr := r.NewCryostatWithSecurityOptions() + cr.Spec.SecurityOptions.DatabaseSecurityContext = nil + cr.Spec.SecurityOptions.StorageSecurityContext = nil + cr.Spec.SecurityOptions.AuthProxySecurityContext = nil + cr.Spec.SecurityOptions.AgentProxySecurityContext = nil + return cr +} + func (r *TestResources) NewCryostatWithReportSecurityOptionsV1Beta1() *operatorv1beta1.Cryostat { cr := r.NewCryostatV1Beta1() nonRoot := true diff --git a/internal/test/reconciler.go b/internal/test/reconciler.go index 00833293..8b339d8b 100644 --- a/internal/test/reconciler.go +++ b/internal/test/reconciler.go @@ -35,6 +35,7 @@ type TestReconcilerConfig struct { EnvDatabaseImageTag *string EnvGrafanaImageTag *string EnvReportsImageTag *string + EnvAgentProxyImageTag *string GeneratedPasswords []string ControllerBuilder *TestCtrlBuilder CertManagerMissing bool @@ -82,6 +83,9 @@ func newTestOSUtils(config *TestReconcilerConfig) *testOSUtils { if config.EnvOpenShiftOAuthProxyImageTag != nil { envs["RELATED_IMAGE_OPENSHIFT_OAUTH_PROXY"] = *config.EnvOpenShiftOAuthProxyImageTag } + if config.EnvAgentProxyImageTag != nil { + envs["RELATED_IMAGE_AGENT_PROXY"] = *config.EnvAgentProxyImageTag + } return &testOSUtils{envs: envs, passwords: config.GeneratedPasswords} } diff --git a/internal/test/resources.go b/internal/test/resources.go index 53b9c25d..03dd9a51 100644 --- a/internal/test/resources.go +++ b/internal/test/resources.go @@ -275,6 +275,28 @@ func (r *TestResources) NewCryostatWithReportsSvc() *model.CryostatInstance { return cr } +func (r *TestResources) NewCryostatWithAgentSvc() *model.CryostatInstance { + svcType := corev1.ServiceTypeNodePort + httpPort := int32(8080) + cr := r.NewCryostat() + cr.Spec.ServiceOptions = &operatorv1beta2.ServiceConfigList{ + AgentConfig: &operatorv1beta2.AgentServiceConfig{ + HTTPPort: &httpPort, + ServiceConfig: operatorv1beta2.ServiceConfig{ + ServiceType: &svcType, + Annotations: map[string]string{ + "my/custom": "annotation", + }, + Labels: map[string]string{ + "my": "label", + "app": "somethingelse", + }, + }, + }, + } + return cr +} + func (r *TestResources) NewCryostatWithCoreNetworkOptions() *model.CryostatInstance { cr := r.NewCryostat() cr.Spec.NetworkOptions = &operatorv1beta2.NetworkConfigurationList{ @@ -446,6 +468,46 @@ func (r *TestResources) NewCryostatWithResources() *model.CryostatInstance { corev1.ResourceMemory: resource.MustParse("64Mi"), }, }, + ObjectStorageResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("600m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("300m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + }, + DatabaseResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + }, + AuthProxyResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("80m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("40m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + AgentProxyResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("60m"), + corev1.ResourceMemory: resource.MustParse("160Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("30m"), + corev1.ResourceMemory: resource.MustParse("80Mi"), + }, + }, } return cr } @@ -471,6 +533,30 @@ func (r *TestResources) NewCryostatWithLowResourceLimit() *model.CryostatInstanc corev1.ResourceMemory: resource.MustParse("32Mi"), }, }, + ObjectStorageResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("40m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + }, + DatabaseResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("32Mi"), + }, + }, + AuthProxyResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("20m"), + corev1.ResourceMemory: resource.MustParse("40Mi"), + }, + }, + AgentProxyResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("15m"), + corev1.ResourceMemory: resource.MustParse("45Mi"), + }, + }, } return cr } @@ -567,6 +653,34 @@ func (r *TestResources) NewCryostatWithSecurityOptions() *model.CryostatInstance }, RunAsUser: &runAsUser, }, + StorageSecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: &privEscalation, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{}, + }, + RunAsUser: &runAsUser, + }, + DatabaseSecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: &privEscalation, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{}, + }, + RunAsUser: &runAsUser, + }, + AuthProxySecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: &privEscalation, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{}, + }, + RunAsUser: &runAsUser, + }, + AgentProxySecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: &privEscalation, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{}, + }, + RunAsUser: &runAsUser, + }, } return cr } @@ -646,20 +760,10 @@ func (r *TestResources) NewCryostatWithAdditionalMetadata() *model.CryostatInsta } func (r *TestResources) NewCryostatService() *corev1.Service { - c := true return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: r.Name, Namespace: r.Namespace, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: operatorv1beta2.GroupVersion.String(), - Kind: "Cryostat", - Name: r.Name, - UID: "", - Controller: &c, - }, - }, Labels: map[string]string{ "app": r.Name, "component": "cryostat", @@ -682,58 +786,38 @@ func (r *TestResources) NewCryostatService() *corev1.Service { } } -func (r *TestResources) NewGrafanaService() *corev1.Service { - c := true +func (r *TestResources) NewReportsService() *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: r.Name + "-grafana", + Name: r.Name + "-reports", Namespace: r.Namespace, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: operatorv1beta2.GroupVersion.String(), - Kind: "Cryostat", - Name: r.Name, - UID: "", - Controller: &c, - }, - }, Labels: map[string]string{ "app": r.Name, - "component": "cryostat", + "component": "reports", }, }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{ "app": r.Name, - "component": "cryostat", + "component": "reports", }, Ports: []corev1.ServicePort{ { Name: "http", - Port: 3000, - TargetPort: intstr.FromInt(3000), + Port: 10000, + TargetPort: intstr.FromInt(10000), }, }, }, } } -func (r *TestResources) NewCommandService() *corev1.Service { - c := true +func (r *TestResources) NewAgentProxyService() *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: r.Name + "-command", + Name: r.Name + "-agent", Namespace: r.Namespace, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: operatorv1beta2.GroupVersion.String(), - Kind: "Cryostat", - Name: r.Name, - UID: "", - Controller: &c, - }, - }, Labels: map[string]string{ "app": r.Name, "component": "cryostat", @@ -748,45 +832,8 @@ func (r *TestResources) NewCommandService() *corev1.Service { Ports: []corev1.ServicePort{ { Name: "http", - Port: 10001, - TargetPort: intstr.FromInt(10001), - }, - }, - }, - } -} - -func (r *TestResources) NewReportsService() *corev1.Service { - c := true - return &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: r.Name + "-reports", - Namespace: r.Namespace, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: operatorv1beta2.GroupVersion.String(), - Kind: "Cryostat", - Name: r.Name + "-reports", - UID: "", - Controller: &c, - }, - }, - Labels: map[string]string{ - "app": r.Name, - "component": "reports", - }, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - Selector: map[string]string{ - "app": r.Name, - "component": "reports", - }, - Ports: []corev1.ServicePort{ - { - Name: "http", - Port: 10000, - TargetPort: intstr.FromInt(10000), + Port: 8443, + TargetPort: intstr.FromInt(8443), }, }, }, @@ -823,6 +870,21 @@ func (r *TestResources) NewCustomizedReportsService() *corev1.Service { return svc } +func (r *TestResources) NewCustomizedAgentService() *corev1.Service { + svc := r.NewAgentProxyService() + svc.Spec.Type = corev1.ServiceTypeNodePort + svc.Spec.Ports[0].Port = 8080 + svc.Annotations = map[string]string{ + "my/custom": "annotation", + } + svc.Labels = map[string]string{ + "app": r.Name, + "component": "cryostat", + "my": "label", + } + return svc +} + func (r *TestResources) NewTestService() *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -1022,6 +1084,32 @@ func (r *TestResources) NewReportsCert() *certv1.Certificate { } } +func (r *TestResources) NewAgentProxyCert() *certv1.Certificate { + return &certv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.Name + "-agent-proxy", + Namespace: r.Namespace, + }, + Spec: certv1.CertificateSpec{ + CommonName: fmt.Sprintf(r.Name+"-agent.%s.svc", r.Namespace), + DNSNames: []string{ + r.Name + "-agent", + fmt.Sprintf(r.Name+"-agent.%s.svc", r.Namespace), + fmt.Sprintf(r.Name+"-agent.%s.svc.cluster.local", r.Namespace), + }, + SecretName: r.Name + "-agent-tls", + IssuerRef: certMeta.ObjectReference{ + Name: r.Name + "-ca", + }, + Usages: []certv1.KeyUsage{ + certv1.UsageDigitalSignature, + certv1.UsageKeyEncipherment, + certv1.UsageServerAuth, + }, + }, + } +} + func (r *TestResources) NewCACert() *certv1.Certificate { return &certv1.Certificate{ ObjectMeta: metav1.ObjectMeta{ @@ -1255,6 +1343,17 @@ func (r *TestResources) NewAuthProxyPorts() []corev1.ContainerPort { } } +func (r *TestResources) NewAgentProxyPorts() []corev1.ContainerPort { + return []corev1.ContainerPort{ + { + ContainerPort: 8443, + }, + { + ContainerPort: 8081, + }, + } +} + func (r *TestResources) NewCoreEnvironmentVariables(reportsUrl string, ingress bool, emptyDir bool, hasPortConfig bool, builtInDiscoveryDisabled bool, builtInPortConfigDisabled bool, dbSecretProvided bool) []corev1.EnvVar { envs := []corev1.EnvVar{ @@ -1634,6 +1733,10 @@ func (r *TestResources) NewAuthProxyEnvironmentVariables(authOptions *operatorv1 return envs } +func (r *TestResources) NewAgentProxyEnvironmentVariables() []corev1.EnvVar { + return []corev1.EnvVar{} +} + func (r *TestResources) NewAuthProxyEnvFromSource() []corev1.EnvFromSource { return []corev1.EnvFromSource{ { @@ -1647,6 +1750,10 @@ func (r *TestResources) NewAuthProxyEnvFromSource() []corev1.EnvFromSource { } } +func (r *TestResources) NewAgentProxyEnvFromSource() []corev1.EnvFromSource { + return []corev1.EnvFromSource{} +} + func (r *TestResources) NewCoreEnvFromSource() []corev1.EnvFromSource { envsFrom := []corev1.EnvFromSource{} return envsFrom @@ -1789,6 +1896,12 @@ func (r *TestResources) NewAuthProxyArguments(authOptions *operatorv1beta2.Autho return args, nil } +func (r *TestResources) NewAgentProxyCommand() []string { + return []string{ + "nginx", "-c", "/etc/nginx-cryostat/nginx.conf", "-g", "daemon off;", + } +} + func (r *TestResources) NewCoreVolumeMounts() []corev1.VolumeMount { mounts := []corev1.VolumeMount{ { @@ -1871,6 +1984,26 @@ func (r *TestResources) NewAuthProxyVolumeMounts(authOptions *operatorv1beta2.Au return mounts } +func (r *TestResources) NewAgentProxyVolumeMounts() []corev1.VolumeMount { + mounts := []corev1.VolumeMount{} + if r.TLS { + mounts = append(mounts, corev1.VolumeMount{ + Name: "agent-proxy-tls-secret", + MountPath: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s-agent-tls", r.Name), + ReadOnly: true, + }) + } + + mounts = append(mounts, + corev1.VolumeMount{ + Name: "agent-proxy-config", + MountPath: "/etc/nginx-cryostat", + ReadOnly: true, + }) + + return mounts +} + func (r *TestResources) NewReportsVolumeMounts() []corev1.VolumeMount { mounts := []corev1.VolumeMount{} if r.TLS { @@ -2001,6 +2134,18 @@ func (r *TestResources) NewAuthProxyLivenessProbe() *corev1.Probe { } } +func (r *TestResources) NewAgentProxyLivenessProbe() *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.IntOrString{IntVal: 8081}, + Path: "/healthz", + Scheme: corev1.URISchemeHTTP, + }, + }, + } +} + func (r *TestResources) NewReportsLivenessProbe() *corev1.Probe { protocol := corev1.URISchemeHTTPS if !r.TLS { @@ -2234,7 +2379,28 @@ func (r *TestResources) newVolumes(certProjections []corev1.VolumeProjection) [] Name: "auth-proxy-tls-secret", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: r.Name + "-tls", + SecretName: r.Name + "-tls", + DefaultMode: &readOnlymode, + }, + }, + }, + corev1.Volume{ + Name: "agent-proxy-tls-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: r.Name + "-agent-tls", + DefaultMode: &readOnlymode, + }, + }, + }, + corev1.Volume{ + Name: "agent-proxy-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: r.Name + "-agent-proxy", + }, + DefaultMode: &readOnlymode, }, }, }, @@ -2375,6 +2541,13 @@ func (r *TestResources) NewAuthProxySecurityContext(cr *model.CryostatInstance) return r.commonDefaultSecurityContext() } +func (r *TestResources) NewAgentProxySecurityContext(cr *model.CryostatInstance) *corev1.SecurityContext { + if cr.Spec.SecurityOptions != nil && cr.Spec.SecurityOptions.AgentProxySecurityContext != nil { + return cr.Spec.SecurityOptions.AgentProxySecurityContext + } + return r.commonDefaultSecurityContext() +} + func (r *TestResources) NewReportSecurityContext(cr *model.CryostatInstance) *corev1.SecurityContext { if cr.Spec.ReportOptions != nil && cr.Spec.ReportOptions.SecurityOptions != nil && cr.Spec.ReportOptions.SecurityOptions.ReportsSecurityContext != nil { return cr.Spec.ReportOptions.SecurityOptions.ReportsSecurityContext @@ -3057,6 +3230,26 @@ func (r *TestResources) NewAuthProxyContainerResource(cr *model.CryostatInstance return resources } +func (r *TestResources) NewAgentProxyContainerResource(cr *model.CryostatInstance) *corev1.ResourceRequirements { + resources := &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("25m"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + } + + if cr.Spec.Resources != nil && cr.Spec.Resources.AgentProxyResources.Requests != nil { + resources.Requests = cr.Spec.Resources.AgentProxyResources.Requests + } + + if cr.Spec.Resources != nil && cr.Spec.Resources.AgentProxyResources.Limits != nil { + resources.Limits = cr.Spec.Resources.AgentProxyResources.Limits + checkWithLimit(resources.Requests, resources.Limits) + } + + return resources +} + func (r *TestResources) NewReportContainerResource(cr *model.CryostatInstance) *corev1.ResourceRequirements { resources := &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ @@ -3100,6 +3293,120 @@ func (r *TestResources) NewLockConfigMap() *corev1.ConfigMap { } } +func (r *TestResources) NewAgentProxyConfigMap() *corev1.ConfigMap { + nginxConf := fmt.Sprintf(` +worker_processes auto; +error_log stderr notice; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /dev/stdout main; + + sendfile on; + tcp_nopush on; + keepalive_timeout 65; + types_hash_max_size 4096; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + server { + listen 8081 ssl; + listen [::]:8181 ssl; + server_name %s-agent.%s.svc; + + ssl_certificate /var/run/secrets/operator.cryostat.io/%s-agent/tls.crt; + ssl_certificate_key /var/run/secrets/operator.cryostat.io/%s-agent/tls.key; + + ssl_session_timeout 5m; + ssl_session_cache shared:SSL:20m; + ssl_session_tickets off; + + ssl_dhparam /etc/nginx-cryostat/dhparam.pem; + + # intermediate configuration + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; + ssl_prefer_server_ciphers off; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + ssl_trusted_certificate /var/run/secrets/operator.cryostat.io/%s-agent/ca.crt; + + # Client certificate authentication + ssl_client_certificate /var/run/secrets/operator.cryostat.io/%s-agent/ca.crt; + ssl_verify_client on; + + location /api/v2.2/discovery/ { + proxy_pass http://127.0.0.1:8181$request_uri; + } + + location /api/v2.2/credentials/ { + proxy_pass http://127.0.0.1:8181$request_uri; + } + + location /api/beta/recordings/ { + proxy_pass http://127.0.0.1:8181$request_uri; + } + + location /health/ { + proxy_pass http://127.0.0.1:8181$request_uri; + } + + location / { + return 404; + } + } + + # Heatlh Check + server { + listen 8081; + + location = /healthz { + return 200; + } + + location / { + return 404; + } + } +}`, r.Name, r.Namespace, r.Name, r.Name, r.Name, r.Name) + + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.Name + "-agent-proxy", + Namespace: r.Namespace, + }, + Data: map[string]string{ + "nginx.conf": nginxConf, + "dhparam.pem": `-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz ++8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a +87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 +YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi +7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD +ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== +-----END DH PARAMETERS-----`, + }, + } +} + func (r *TestResources) getClusterUniqueName() string { return "cryostat-" + r.clusterUniqueSuffix("") } From f81fbf4d2dc86aa739001a215550006cef92a015 Mon Sep 17 00:00:00 2001 From: Elliott Baron Date: Fri, 20 Sep 2024 16:29:34 -0400 Subject: [PATCH 3/6] Allow proxy to work without TLS --- api/v1beta2/cryostat_types.go | 2 +- ...yostat-operator.clusterserviceversion.yaml | 2 +- .../operator.cryostat.io_cryostats.yaml | 2 +- .../bases/operator.cryostat.io_cryostats.yaml | 2 +- .../resource_definitions.go | 68 +++--- internal/controllers/configmaps.go | 196 ++++++++++-------- internal/controllers/constants/constants.go | 5 +- internal/controllers/reconciler.go | 2 +- internal/controllers/reconciler_test.go | 26 ++- internal/controllers/services.go | 7 +- internal/test/resources.go | 146 ++++++++++--- 11 files changed, 290 insertions(+), 168 deletions(-) diff --git a/api/v1beta2/cryostat_types.go b/api/v1beta2/cryostat_types.go index 4148c579..36885a1c 100644 --- a/api/v1beta2/cryostat_types.go +++ b/api/v1beta2/cryostat_types.go @@ -314,7 +314,7 @@ type ReportsServiceConfig struct { // traffic from Cryostat agents to the Cryostat application. type AgentServiceConfig struct { // HTTP port number for the Cryostat agent API service. - // Defaults to 8443. + // Defaults to 8282. // +optional HTTPPort *int32 `json:"httpPort,omitempty"` ServiceConfig `json:",inline"` diff --git a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml index 730c6ead..326f8a19 100644 --- a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml +++ b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml @@ -30,7 +30,7 @@ metadata: capabilities: Seamless Upgrades categories: Monitoring, Developer Tools containerImage: quay.io/cryostat/cryostat-operator:4.0.0-dev - createdAt: "2024-09-20T18:00:33Z" + createdAt: "2024-09-20T20:29:31Z" description: JVM monitoring and profiling tool operatorframework.io/initialization-resource: |- { diff --git a/bundle/manifests/operator.cryostat.io_cryostats.yaml b/bundle/manifests/operator.cryostat.io_cryostats.yaml index 05092905..5eceac38 100644 --- a/bundle/manifests/operator.cryostat.io_cryostats.yaml +++ b/bundle/manifests/operator.cryostat.io_cryostats.yaml @@ -9142,7 +9142,7 @@ spec: httpPort: description: |- HTTP port number for the Cryostat agent API service. - Defaults to 8443. + Defaults to 8282. format: int32 type: integer labels: diff --git a/config/crd/bases/operator.cryostat.io_cryostats.yaml b/config/crd/bases/operator.cryostat.io_cryostats.yaml index a963d40b..109b114f 100644 --- a/config/crd/bases/operator.cryostat.io_cryostats.yaml +++ b/config/crd/bases/operator.cryostat.io_cryostats.yaml @@ -9129,7 +9129,7 @@ spec: httpPort: description: |- HTTP port number for the Cryostat agent API service. - Defaults to 8443. + Defaults to 8282. format: int32 type: integer labels: diff --git a/internal/controllers/common/resource_definitions/resource_definitions.go b/internal/controllers/common/resource_definitions/resource_definitions.go index 672b24cd..a93eddf5 100644 --- a/internal/controllers/common/resource_definitions/resource_definitions.go +++ b/internal/controllers/common/resource_definitions/resource_definitions.go @@ -271,6 +271,7 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima NewStorageContainer(cr, imageTags.StorageImageTag, tls), newDatabaseContainer(cr, imageTags.DatabaseImageTag, tls), *authProxy, + newAgentProxyContainer(cr, imageTags.AgentProxyImageTag, tls), } volumes := newVolumeForCR(cr) @@ -303,8 +304,6 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima } if tls != nil { - containers = append(containers, newAgentProxyContainer(cr, imageTags.AgentProxyImageTag, tls)) - volSources = append(volSources, corev1.VolumeProjection{ // Add Cryostat self-signed CA Secret: &corev1.SecretProjection{ @@ -340,17 +339,6 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima }, }, }, - corev1.Volume{ - Name: "agent-proxy-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: cr.Name + "-agent-proxy", - }, - DefaultMode: &readOnlyMode, - }, - }, - }, corev1.Volume{ Name: "keystore", VolumeSource: corev1.VolumeSource{ @@ -378,7 +366,21 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima }, }, } - volumes = append(volumes, certVolume) + + // Add agent proxy config map as a volume + agentProxyVolume := corev1.Volume{ + Name: "agent-proxy-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cr.Name + "-agent-proxy", + }, + DefaultMode: &readOnlyMode, + }, + }, + } + + volumes = append(volumes, certVolume, agentProxyVolume) if !openshift { // if not deploying openshift-oauth-proxy then we must be deploying oauth2_proxy instead @@ -1564,7 +1566,23 @@ func newAgentProxyContainer(cr *model.CryostatInstance, imageTag string, tls *TL } } - healthCheckPort := int32(8081) + // Mount the config map containing the nginx.conf (and DH params if TLS is enabled) + mounts := []corev1.VolumeMount{ + { + Name: "agent-proxy-config", + MountPath: constants.AgentProxyConfigFilePath, + ReadOnly: true, + }, + } + if tls != nil { + // Mount the TLS secret for the agent proxy + mounts = append(mounts, corev1.VolumeMount{ + Name: "agent-proxy-tls-secret", + MountPath: "/var/run/secrets/operator.cryostat.io/" + tls.AgentProxySecret, + ReadOnly: true, + }) + } + return corev1.Container{ Name: cr.Name + "-agent-proxy", Image: imageTag, @@ -1574,37 +1592,27 @@ func newAgentProxyContainer(cr *model.CryostatInstance, imageTag string, tls *TL ContainerPort: constants.AgentProxyContainerPort, }, { - ContainerPort: healthCheckPort, + ContainerPort: constants.AgentProxyHealthPort, }, }, + // Override the command to run nginx pointed at our config file. See: + // https://github.com/sclorg/nginx-container/blob/e7d8db9bc5299a4c4e254f8a82e917c7c136468b/1.24/README.md#direct-usage-with-a-mounted-directory Command: []string{ "nginx", "-c", fmt.Sprintf("%s/%s", constants.AgentProxyConfigFilePath, constants.AgentProxyConfigFileName), "-g", "daemon off;"}, - // Can't use HTTP probe since the port is not exposed over the network LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", - Port: intstr.FromInt32(healthCheckPort), + Port: intstr.FromInt32(constants.AgentProxyHealthPort), Scheme: corev1.URISchemeHTTP, }, }, }, SecurityContext: securityContext, Resources: *newAgentProxyContainerResource(cr), - VolumeMounts: []corev1.VolumeMount{ - { - Name: "agent-proxy-tls-secret", - MountPath: "/var/run/secrets/operator.cryostat.io/" + tls.AgentProxySecret, - ReadOnly: true, - }, - { - Name: "agent-proxy-config", - MountPath: constants.AgentProxyConfigFilePath, - ReadOnly: true, - }, - }, + VolumeMounts: mounts, } } diff --git a/internal/controllers/configmaps.go b/internal/controllers/configmaps.go index 7f3ba423..f7254035 100644 --- a/internal/controllers/configmaps.go +++ b/internal/controllers/configmaps.go @@ -86,86 +86,96 @@ type alphaConfigUpstream struct { } func (r *Reconciler) reconcileOAuth2ProxyConfig(ctx context.Context, cr *model.CryostatInstance, tls *resources.TLSConfig) error { + bindHost := "0.0.0.0" + immutable := true + cfg := &oauth2ProxyAlphaConfig{ + Server: alphaConfigServer{}, + UpstreamConfig: alphaConfigUpstreamConfig{ProxyRawPath: true, Upstreams: []alphaConfigUpstream{ + { + Id: "cryostat", + Path: "/", + Uri: fmt.Sprintf("http://localhost:%d", constants.CryostatHTTPContainerPort), + }, + { + Id: "grafana", + Path: "/grafana/", + Uri: fmt.Sprintf("http://localhost:%d", constants.GrafanaContainerPort), + }, + { + Id: "storage", + Path: "^/storage/(.*)$", + RewriteTarget: "/$1", + Uri: fmt.Sprintf("http://localhost:%d", constants.StoragePort), + PassHostHeader: false, + ProxyWebSockets: false, + }, + }}, + Providers: []alphaConfigProvider{{Id: "dummy", Name: "Unused - Sign In Below", ClientId: "CLIENT_ID", ClientSecret: "CLIENT_SECRET", Provider: "google"}}, + } + + if tls != nil { + cfg.Server.SecureBindAddress = fmt.Sprintf("https://%s:%d", bindHost, constants.AuthProxyHttpContainerPort) + cfg.Server.TLS = proxyTLS{ + Key: tlsSecretSource{ + FromFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.CryostatSecret, corev1.TLSPrivateKeyKey), + }, + Cert: tlsSecretSource{ + FromFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.CryostatSecret, corev1.TLSCertKey), + }, + } + } else { + cfg.Server.BindAddress = fmt.Sprintf("http://%s:%d", bindHost, constants.AuthProxyHttpContainerPort) + } + + data := make(map[string]string) + json, err := json.Marshal(cfg) + if err != nil { + return err + } + data[resources.OAuth2ConfigFileName] = string(json) cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: cr.Name + "-oauth2-proxy-cfg", Namespace: cr.InstallNamespace, }, + Immutable: &immutable, + Data: data, } if r.IsOpenShift { return r.deleteConfigMap(ctx, cm) + } else { + return r.createOrUpdateConfigMap(ctx, cm, cr.Object, func() error { + return nil + }) } - - return r.createOrUpdateConfigMap(ctx, cm, cr.Object, func() error { - bindHost := "0.0.0.0" - immutable := true - cfg := &oauth2ProxyAlphaConfig{ - Server: alphaConfigServer{}, - UpstreamConfig: alphaConfigUpstreamConfig{ProxyRawPath: true, Upstreams: []alphaConfigUpstream{ - { - Id: "cryostat", - Path: "/", - Uri: fmt.Sprintf("http://localhost:%d", constants.CryostatHTTPContainerPort), - }, - { - Id: "grafana", - Path: "/grafana/", - Uri: fmt.Sprintf("http://localhost:%d", constants.GrafanaContainerPort), - }, - { - Id: "storage", - Path: "^/storage/(.*)$", - RewriteTarget: "/$1", - Uri: fmt.Sprintf("http://localhost:%d", constants.StoragePort), - PassHostHeader: false, - ProxyWebSockets: false, - }, - }}, - Providers: []alphaConfigProvider{{Id: "dummy", Name: "Unused - Sign In Below", ClientId: "CLIENT_ID", ClientSecret: "CLIENT_SECRET", Provider: "google"}}, - } - - if tls != nil { - cfg.Server.SecureBindAddress = fmt.Sprintf("https://%s:%d", bindHost, constants.AuthProxyHttpContainerPort) - cfg.Server.TLS = proxyTLS{ - Key: tlsSecretSource{ - FromFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.CryostatSecret, corev1.TLSPrivateKeyKey), - }, - Cert: tlsSecretSource{ - FromFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.CryostatSecret, corev1.TLSCertKey), - }, - } - } else { - cfg.Server.BindAddress = fmt.Sprintf("http://%s:%d", bindHost, constants.AuthProxyHttpContainerPort) - } - - data := make(map[string]string) - json, err := json.Marshal(cfg) - if err != nil { - return err - } - data[resources.OAuth2ConfigFileName] = string(json) - - cm.Immutable = &immutable - cm.Data = data - return nil - }) } type nginxConfParams struct { - ServerName string - TLSCertFile string - TLSKeyFile string - CACertFile string - DHParamFile string + // Hostname of the server + ServerName string + // Whether TLS is enabled + TLSEnabled bool + // Path to certificate for HTTPS + TLSCertFile string + // Path to private key for HTTPS + TLSKeyFile string + // Path to CA certificate + CACertFile string + // Diffie-Hellman parameters file + DHParamFile string + // Nginx proxy container port ContainerPort int32 - CryostatPort int32 - AllowedPaths []string + // Nginx health container port + HealthPort int32 + // Cryostat HTTP container port + CryostatPort int32 + // Only these path prefixes will be proxied, others will return 404 + AllowedPathPrefixes []string } // Reference: https://ssl-config.mozilla.org -var nginxConfTemplate = template.Must(template.New("").Parse(` -worker_processes auto; +var nginxConfTemplate = template.Must(template.New("").Parse(`worker_processes auto; error_log stderr notice; pid /run/nginx.pid; @@ -192,9 +202,11 @@ http { default_type application/octet-stream; server { + server_name {{ .ServerName }}; + + {{ if .TLSEnabled -}} listen {{ .ContainerPort }} ssl; listen [::]:{{ .ContainerPort }} ssl; - server_name {{ .ServerName }}; ssl_certificate {{ .TLSCertFile }}; ssl_certificate_key {{ .TLSKeyFile }}; @@ -223,7 +235,14 @@ http { ssl_client_certificate {{ .CACertFile }}; ssl_verify_client on; - {{ range .AllowedPaths -}} + {{- else -}} + + listen {{ .ContainerPort }}; + listen [::]:{{ .ContainerPort }}; + + {{- end }} + + {{ range .AllowedPathPrefixes -}} location {{ . }} { proxy_pass http://127.0.0.1:{{ $.CryostatPort }}$request_uri; } @@ -237,7 +256,8 @@ http { # Heatlh Check server { - listen 8081; + listen {{ .HealthPort }}; + listen [::]:{{ .HealthPort }}; location = /healthz { return 200; @@ -249,8 +269,10 @@ http { } }`)) -// From https://ssl-config.mozilla.org/ffdhe2048.txt -const dhParams = `-----BEGIN DH PARAMETERS----- +const ( + dhFileName = "dhparam.pem" + // From https://ssl-config.mozilla.org/ffdhe2048.txt + dhParams = `-----BEGIN DH PARAMETERS----- MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz +8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a 87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 @@ -258,6 +280,7 @@ YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi 7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== -----END DH PARAMETERS-----` +) func (r *Reconciler) reconcileAgentProxyConfig(ctx context.Context, cr *model.CryostatInstance, tls *resources.TLSConfig) error { cm := &corev1.ConfigMap{ @@ -267,39 +290,44 @@ func (r *Reconciler) reconcileAgentProxyConfig(ctx context.Context, cr *model.Cr }, } - if tls == nil { // TODO make this work without TLS - return r.deleteConfigMap(ctx, cm) - } - + data := map[string]string{} buf := &bytes.Buffer{} params := &nginxConfParams{ ServerName: fmt.Sprintf("%s-agent.%s.svc", cr.Name, cr.InstallNamespace), - TLSCertFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSCertKey), - TLSKeyFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSPrivateKeyKey), - CACertFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, constants.CAKey), - DHParamFile: fmt.Sprintf("%s/%s", constants.AgentProxyConfigFilePath, constants.AgentProxyDHFileName), ContainerPort: constants.AgentProxyContainerPort, + HealthPort: constants.AgentProxyHealthPort, CryostatPort: constants.CryostatHTTPContainerPort, - AllowedPaths: []string{ + AllowedPathPrefixes: []string{ "/api/v2.2/discovery/", "/api/v2.2/credentials/", "/api/beta/recordings/", "/health/", }, } + if tls != nil { + params.TLSEnabled = true + params.TLSCertFile = fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSCertKey) + params.TLSKeyFile = fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSPrivateKeyKey) + params.CACertFile = fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, constants.CAKey) + params.DHParamFile = fmt.Sprintf("%s/%s", constants.AgentProxyConfigFilePath, dhFileName) + + // Add Diffie-Hellman parameters to config map + data[dhFileName] = dhParams + } + + // Create an nginx.conf where: + // 1. If TLS is enabled, requires client certificate authentication against our CA + // 2. Proxies only those API endpoints required by the agent err := nginxConfTemplate.Execute(buf, params) if err != nil { return err } - return r.createOrUpdateConfigMap(ctx, cm, cr.Object, func() error { - immutable := true // TODO breaks update - cm.Immutable = &immutable + // Add generated nginx.conf to config map + data[constants.AgentProxyConfigFileName] = buf.String() - cm.Data = map[string]string{ - constants.AgentProxyConfigFileName: buf.String(), - constants.AgentProxyDHFileName: dhParams, - } + return r.createOrUpdateConfigMap(ctx, cm, cr.Object, func() error { + cm.Data = data return nil }) } diff --git a/internal/controllers/constants/constants.go b/internal/controllers/constants/constants.go index 37c2f723..59b7bd9b 100644 --- a/internal/controllers/constants/constants.go +++ b/internal/controllers/constants/constants.go @@ -27,8 +27,8 @@ const ( ReportsContainerPort int32 = 10000 StoragePort int32 = 8333 DatabasePort int32 = 5432 - AgentProxyContainerPort int32 = 8443 - AgentProxyHealthPort int32 = 8081 + AgentProxyContainerPort int32 = 8282 + AgentProxyHealthPort int32 = 8281 LoopbackAddress string = "127.0.0.1" OperatorNamePrefix string = "cryostat-operator-" OperatorDeploymentName string = "cryostat-operator-controller" @@ -46,7 +46,6 @@ const ( AgentProxyConfigFilePath string = "/etc/nginx-cryostat" AgentProxyConfigFileName string = "nginx.conf" - AgentProxyDHFileName string = "dhparam.pem" // TODO non-exported targetNamespaceCRLabelPrefix = "operator.cryostat.io/" TargetNamespaceCRNameLabel = targetNamespaceCRLabelPrefix + "name" diff --git a/internal/controllers/reconciler.go b/internal/controllers/reconciler.go index 4474d4f6..37c5226f 100644 --- a/internal/controllers/reconciler.go +++ b/internal/controllers/reconciler.go @@ -269,7 +269,7 @@ func (r *Reconciler) reconcileCryostat(ctx context.Context, cr *model.CryostatIn if err != nil { return requeueIfIngressNotReady(reqLogger, err) } - err = r.reconcileAgentService(ctx, cr, tlsConfig, serviceSpecs) + err = r.reconcileAgentService(ctx, cr) if err != nil { return reconcile.Result{}, err } diff --git a/internal/controllers/reconciler_test.go b/internal/controllers/reconciler_test.go index f4dc86d4..612c69f4 100644 --- a/internal/controllers/reconciler_test.go +++ b/internal/controllers/reconciler_test.go @@ -1264,6 +1264,9 @@ func (c *controllerTest) commonTests() { It("should create routes with edge TLS termination", func() { t.expectRoutes() }) + It("should create the agent proxy config map", func() { + t.expectAgentProxyConfigMap() + }) }) Context("with cert-manager not configured in CR", func() { BeforeEach(func() { @@ -1286,6 +1289,9 @@ func (c *controllerTest) commonTests() { t.checkConditionPresent(operatorv1beta2.ConditionTypeTLSSetupComplete, metav1.ConditionTrue, "AllCertificatesReady") }) + It("should create the agent proxy config map", func() { + t.expectAgentProxyConfigMap() + }) }) Context("with DISABLE_SERVICE_TLS=true", func() { BeforeEach(func() { @@ -1315,6 +1321,9 @@ func (c *controllerTest) commonTests() { t.checkConditionPresent(operatorv1beta2.ConditionTypeTLSSetupComplete, metav1.ConditionTrue, "CertManagerDisabled") }) + It("should create the agent proxy config map", func() { + t.expectAgentProxyConfigMap() + }) }) Context("Disable cert-manager after being enabled", func() { BeforeEach(func() { @@ -1342,6 +1351,9 @@ func (c *controllerTest) commonTests() { t.checkConditionPresent(operatorv1beta2.ConditionTypeTLSSetupComplete, metav1.ConditionTrue, "CertManagerDisabled") }) + It("should create the agent proxy config map", func() { + t.expectAgentProxyConfigMap() + }) }) Context("Enable cert-manager after being disabled", func() { BeforeEach(func() { @@ -1373,6 +1385,9 @@ func (c *controllerTest) commonTests() { t.checkConditionPresent(operatorv1beta2.ConditionTypeTLSSetupComplete, metav1.ConditionTrue, "AllCertificatesReady") }) + It("should create the agent proxy config map", func() { + t.expectAgentProxyConfigMap() + }) }) Context("cert-manager missing", func() { JustBeforeEach(func() { @@ -2734,6 +2749,7 @@ func (t *cryostatTestInput) expectAgentProxyConfigMap() { Expect(err).ToNot(HaveOccurred()) t.checkMetadata(cm, expected) + Expect(cm.Data).To(Equal(expected.Data)) } func (t *cryostatTestInput) expectPVC(expectedPVC *corev1.PersistentVolumeClaim) { @@ -2991,6 +3007,7 @@ func (t *cryostatTestInput) checkMainPodTemplate(deployment *appsv1.Deployment, Expect(template.Spec.SecurityContext).To(Equal(t.NewPodSecurityContext(cr))) // Check that the networking environment variables are set correctly + Expect(len(template.Spec.Containers)).To(Equal(7)) coreContainer := template.Spec.Containers[0] port := int32(10000) if cr.Spec.ServiceOptions != nil && cr.Spec.ServiceOptions.ReportsConfig != nil && @@ -3045,12 +3062,9 @@ func (t *cryostatTestInput) checkMainPodTemplate(deployment *appsv1.Deployment, authProxyContainer := template.Spec.Containers[5] t.checkAuthProxyContainer(&authProxyContainer, t.NewAuthProxyContainerResource(cr), t.NewAuthProxySecurityContext(cr), cr.Spec.AuthorizationOptions) - // TODO make this work without TLS - if t.TLS { - // Check that Agent Proxy is configured properly - agentProxyContainer := template.Spec.Containers[6] - t.checkAgentProxyContainer(&agentProxyContainer, t.NewAgentProxyContainerResource(cr), t.NewAgentProxySecurityContext(cr)) - } + // Check that Agent Proxy is configured properly + agentProxyContainer := template.Spec.Containers[6] + t.checkAgentProxyContainer(&agentProxyContainer, t.NewAgentProxyContainerResource(cr), t.NewAgentProxySecurityContext(cr)) // Check that the proper Service Account is set Expect(template.Spec.ServiceAccountName).To(Equal(t.Name)) diff --git a/internal/controllers/services.go b/internal/controllers/services.go index 75697c2a..58ec16f5 100644 --- a/internal/controllers/services.go +++ b/internal/controllers/services.go @@ -111,8 +111,7 @@ func (r *Reconciler) reconcileReportsService(ctx context.Context, cr *model.Cryo return nil } -func (r *Reconciler) reconcileAgentService(ctx context.Context, cr *model.CryostatInstance, - tls *resource_definitions.TLSConfig, specs *resource_definitions.ServiceSpecs) error { +func (r *Reconciler) reconcileAgentService(ctx context.Context, cr *model.CryostatInstance) error { config := configureAgentService(cr) svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -121,10 +120,6 @@ func (r *Reconciler) reconcileAgentService(ctx context.Context, cr *model.Cryost }, } - if tls == nil { - // Delete service if it exists - return r.deleteService(ctx, svc) - } return r.createOrUpdateService(ctx, svc, cr.Object, &config.ServiceConfig, func() error { svc.Spec.Selector = map[string]string{ "app": cr.Name, diff --git a/internal/test/resources.go b/internal/test/resources.go index 03dd9a51..53c2b2f0 100644 --- a/internal/test/resources.go +++ b/internal/test/resources.go @@ -832,8 +832,8 @@ func (r *TestResources) NewAgentProxyService() *corev1.Service { Ports: []corev1.ServicePort{ { Name: "http", - Port: 8443, - TargetPort: intstr.FromInt(8443), + Port: 8282, + TargetPort: intstr.FromInt(8282), }, }, }, @@ -1346,10 +1346,10 @@ func (r *TestResources) NewAuthProxyPorts() []corev1.ContainerPort { func (r *TestResources) NewAgentProxyPorts() []corev1.ContainerPort { return []corev1.ContainerPort{ { - ContainerPort: 8443, + ContainerPort: 8281, }, { - ContainerPort: 8081, + ContainerPort: 8282, }, } } @@ -2138,7 +2138,7 @@ func (r *TestResources) NewAgentProxyLivenessProbe() *corev1.Probe { return &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Port: intstr.IntOrString{IntVal: 8081}, + Port: intstr.IntOrString{IntVal: 8281}, Path: "/healthz", Scheme: corev1.URISchemeHTTP, }, @@ -2341,6 +2341,17 @@ func (r *TestResources) newVolumes(certProjections []corev1.VolumeProjection) [] }, }, }, + { + Name: "agent-proxy-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: r.Name + "-agent-proxy", + }, + DefaultMode: &readOnlymode, + }, + }, + }, } projs := append([]corev1.VolumeProjection{}, certProjections...) if r.TLS { @@ -2393,17 +2404,6 @@ func (r *TestResources) newVolumes(certProjections []corev1.VolumeProjection) [] }, }, }, - corev1.Volume{ - Name: "agent-proxy-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: r.Name + "-agent-proxy", - }, - DefaultMode: &readOnlymode, - }, - }, - }, ) } @@ -3293,9 +3293,7 @@ func (r *TestResources) NewLockConfigMap() *corev1.ConfigMap { } } -func (r *TestResources) NewAgentProxyConfigMap() *corev1.ConfigMap { - nginxConf := fmt.Sprintf(` -worker_processes auto; +const nginxFormatTLS = `worker_processes auto; error_log stderr notice; pid /run/nginx.pid; @@ -3322,12 +3320,13 @@ http { default_type application/octet-stream; server { - listen 8081 ssl; - listen [::]:8181 ssl; server_name %s-agent.%s.svc; - ssl_certificate /var/run/secrets/operator.cryostat.io/%s-agent/tls.crt; - ssl_certificate_key /var/run/secrets/operator.cryostat.io/%s-agent/tls.key; + listen 8282 ssl; + listen [::]:8282 ssl; + + ssl_certificate /var/run/secrets/operator.cryostat.io/%s-agent-tls/tls.crt; + ssl_certificate_key /var/run/secrets/operator.cryostat.io/%s-agent-tls/tls.key; ssl_session_timeout 5m; ssl_session_cache shared:SSL:20m; @@ -3347,10 +3346,10 @@ http { ssl_stapling on; ssl_stapling_verify on; - ssl_trusted_certificate /var/run/secrets/operator.cryostat.io/%s-agent/ca.crt; + ssl_trusted_certificate /var/run/secrets/operator.cryostat.io/%s-agent-tls/ca.crt; # Client certificate authentication - ssl_client_certificate /var/run/secrets/operator.cryostat.io/%s-agent/ca.crt; + ssl_client_certificate /var/run/secrets/operator.cryostat.io/%s-agent-tls/ca.crt; ssl_verify_client on; location /api/v2.2/discovery/ { @@ -3376,7 +3375,8 @@ http { # Heatlh Check server { - listen 8081; + listen 8281; + listen [::]:8281; location = /healthz { return 200; @@ -3386,15 +3386,81 @@ http { return 404; } } -}`, r.Name, r.Namespace, r.Name, r.Name, r.Name, r.Name) +}` - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: r.Name + "-agent-proxy", - Namespace: r.Namespace, - }, - Data: map[string]string{ - "nginx.conf": nginxConf, +const nginxFormatNoTLS = `worker_processes auto; +error_log stderr notice; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /dev/stdout main; + + sendfile on; + tcp_nopush on; + keepalive_timeout 65; + types_hash_max_size 4096; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + server { + server_name %s-agent.%s.svc; + + listen 8282; + listen [::]:8282; + + location /api/v2.2/discovery/ { + proxy_pass http://127.0.0.1:8181$request_uri; + } + + location /api/v2.2/credentials/ { + proxy_pass http://127.0.0.1:8181$request_uri; + } + + location /api/beta/recordings/ { + proxy_pass http://127.0.0.1:8181$request_uri; + } + + location /health/ { + proxy_pass http://127.0.0.1:8181$request_uri; + } + + location / { + return 404; + } + } + + # Heatlh Check + server { + listen 8281; + listen [::]:8281; + + location = /healthz { + return 200; + } + + location / { + return 404; + } + } +}` + +func (r *TestResources) NewAgentProxyConfigMap() *corev1.ConfigMap { + var data map[string]string + if r.TLS { + data = map[string]string{ + "nginx.conf": fmt.Sprintf(nginxFormatTLS, r.Name, r.Namespace, r.Name, r.Name, r.Name, r.Name), "dhparam.pem": `-----BEGIN DH PARAMETERS----- MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz +8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a @@ -3403,7 +3469,19 @@ YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi 7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== -----END DH PARAMETERS-----`, + } + } else { + data = map[string]string{ + "nginx.conf": fmt.Sprintf(nginxFormatNoTLS, r.Name, r.Namespace), + } + } + + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.Name + "-agent-proxy", + Namespace: r.Namespace, }, + Data: data, } } From af7f01f16b5859193867dfb719e3f474ddcd20ff Mon Sep 17 00:00:00 2001 From: Elliott Baron Date: Wed, 25 Sep 2024 11:32:11 -0400 Subject: [PATCH 4/6] Add exact match location blocks to avoid 301 --- internal/controllers/configmaps.go | 14 ++++++++----- internal/test/resources.go | 32 ++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/internal/controllers/configmaps.go b/internal/controllers/configmaps.go index f7254035..84141cf0 100644 --- a/internal/controllers/configmaps.go +++ b/internal/controllers/configmaps.go @@ -243,7 +243,11 @@ http { {{- end }} {{ range .AllowedPathPrefixes -}} - location {{ . }} { + location {{ . }}/ { + proxy_pass http://127.0.0.1:{{ $.CryostatPort }}$request_uri; + } + + location = {{ . }} { proxy_pass http://127.0.0.1:{{ $.CryostatPort }}$request_uri; } @@ -298,10 +302,10 @@ func (r *Reconciler) reconcileAgentProxyConfig(ctx context.Context, cr *model.Cr HealthPort: constants.AgentProxyHealthPort, CryostatPort: constants.CryostatHTTPContainerPort, AllowedPathPrefixes: []string{ - "/api/v2.2/discovery/", - "/api/v2.2/credentials/", - "/api/beta/recordings/", - "/health/", + "/api/v2.2/discovery", + "/api/v2.2/credentials", + "/api/beta/recordings", + "/health", }, } if tls != nil { diff --git a/internal/test/resources.go b/internal/test/resources.go index 53c2b2f0..e117ffa4 100644 --- a/internal/test/resources.go +++ b/internal/test/resources.go @@ -3356,18 +3356,34 @@ http { proxy_pass http://127.0.0.1:8181$request_uri; } + location = /api/v2.2/discovery { + proxy_pass http://127.0.0.1:8181$request_uri; + } + location /api/v2.2/credentials/ { proxy_pass http://127.0.0.1:8181$request_uri; } + location = /api/v2.2/credentials { + proxy_pass http://127.0.0.1:8181$request_uri; + } + location /api/beta/recordings/ { proxy_pass http://127.0.0.1:8181$request_uri; } + location = /api/beta/recordings { + proxy_pass http://127.0.0.1:8181$request_uri; + } + location /health/ { proxy_pass http://127.0.0.1:8181$request_uri; } + location = /health { + proxy_pass http://127.0.0.1:8181$request_uri; + } + location / { return 404; } @@ -3424,18 +3440,34 @@ http { proxy_pass http://127.0.0.1:8181$request_uri; } + location = /api/v2.2/discovery { + proxy_pass http://127.0.0.1:8181$request_uri; + } + location /api/v2.2/credentials/ { proxy_pass http://127.0.0.1:8181$request_uri; } + location = /api/v2.2/credentials { + proxy_pass http://127.0.0.1:8181$request_uri; + } + location /api/beta/recordings/ { proxy_pass http://127.0.0.1:8181$request_uri; } + location = /api/beta/recordings { + proxy_pass http://127.0.0.1:8181$request_uri; + } + location /health/ { proxy_pass http://127.0.0.1:8181$request_uri; } + location = /health { + proxy_pass http://127.0.0.1:8181$request_uri; + } + location / { return 404; } From fb58f1fa54e5e08bbcbc84863fac58a9369b2865 Mon Sep 17 00:00:00 2001 From: Andrew Azores Date: Thu, 3 Oct 2024 13:20:00 -0400 Subject: [PATCH 5/6] V4 API paths --- internal/controllers/configmaps.go | 4 ++-- internal/test/resources.go | 16 ++++++++-------- internal/test/scorecard/clients.go | 18 +++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/internal/controllers/configmaps.go b/internal/controllers/configmaps.go index 84141cf0..ad671e0f 100644 --- a/internal/controllers/configmaps.go +++ b/internal/controllers/configmaps.go @@ -302,8 +302,8 @@ func (r *Reconciler) reconcileAgentProxyConfig(ctx context.Context, cr *model.Cr HealthPort: constants.AgentProxyHealthPort, CryostatPort: constants.CryostatHTTPContainerPort, AllowedPathPrefixes: []string{ - "/api/v2.2/discovery", - "/api/v2.2/credentials", + "/api/v4/discovery", + "/api/v4/credentials", "/api/beta/recordings", "/health", }, diff --git a/internal/test/resources.go b/internal/test/resources.go index e117ffa4..e576e5b1 100644 --- a/internal/test/resources.go +++ b/internal/test/resources.go @@ -3352,19 +3352,19 @@ http { ssl_client_certificate /var/run/secrets/operator.cryostat.io/%s-agent-tls/ca.crt; ssl_verify_client on; - location /api/v2.2/discovery/ { + location /api/v4/discovery/ { proxy_pass http://127.0.0.1:8181$request_uri; } - location = /api/v2.2/discovery { + location = /api/v4/discovery { proxy_pass http://127.0.0.1:8181$request_uri; } - location /api/v2.2/credentials/ { + location /api/v4/credentials/ { proxy_pass http://127.0.0.1:8181$request_uri; } - location = /api/v2.2/credentials { + location = /api/v4/credentials { proxy_pass http://127.0.0.1:8181$request_uri; } @@ -3436,19 +3436,19 @@ http { listen 8282; listen [::]:8282; - location /api/v2.2/discovery/ { + location /api/v4/discovery/ { proxy_pass http://127.0.0.1:8181$request_uri; } - location = /api/v2.2/discovery { + location = /api/v4/discovery { proxy_pass http://127.0.0.1:8181$request_uri; } - location /api/v2.2/credentials/ { + location /api/v4/credentials/ { proxy_pass http://127.0.0.1:8181$request_uri; } - location = /api/v2.2/credentials { + location = /api/v4/credentials { proxy_pass http://127.0.0.1:8181$request_uri; } diff --git a/internal/test/scorecard/clients.go b/internal/test/scorecard/clients.go index 734a4e42..74b4382c 100644 --- a/internal/test/scorecard/clients.go +++ b/internal/test/scorecard/clients.go @@ -241,7 +241,7 @@ type TargetClient struct { } func (client *TargetClient) List(ctx context.Context) ([]Target, error) { - url := client.Base.JoinPath("/api/v1/targets") + url := client.Base.JoinPath("/api/v4/targets") header := make(http.Header) header.Add("Accept", "*/*") @@ -265,7 +265,7 @@ func (client *TargetClient) List(ctx context.Context) ([]Target, error) { } func (client *TargetClient) Create(ctx context.Context, options *Target) (*Target, error) { - url := client.Base.JoinPath("/api/v2/targets") + url := client.Base.JoinPath("/api/v4/targets") header := make(http.Header) header.Add("Content-Type", "application/x-www-form-urlencoded") header.Add("Accept", "*/*") @@ -296,7 +296,7 @@ type RecordingClient struct { } func (client *RecordingClient) List(ctx context.Context, target *Target) ([]Recording, error) { - url := client.Base.JoinPath(fmt.Sprintf("/api/v3/targets/%d/recordings", target.Id)) + url := client.Base.JoinPath(fmt.Sprintf("/api/v4/targets/%d/recordings", target.Id)) header := make(http.Header) header.Add("Accept", "*/*") @@ -335,7 +335,7 @@ func (client *RecordingClient) Get(ctx context.Context, target *Target, recordin } func (client *RecordingClient) Create(ctx context.Context, target *Target, options *RecordingCreateOptions) (*Recording, error) { - url := client.Base.JoinPath(fmt.Sprintf("/api/v3/targets/%d/recordings", target.Id)) + url := client.Base.JoinPath(fmt.Sprintf("/api/v4/targets/%d/recordings", target.Id)) body := options.ToFormData() header := make(http.Header) header.Add("Content-Type", "application/x-www-form-urlencoded") @@ -361,7 +361,7 @@ func (client *RecordingClient) Create(ctx context.Context, target *Target, optio } func (client *RecordingClient) Archive(ctx context.Context, target *Target, recordingId uint32) (string, error) { - url := client.Base.JoinPath(fmt.Sprintf("/api/v3/targets/%d/recordings/%d", target.Id, recordingId)) + url := client.Base.JoinPath(fmt.Sprintf("/api/v4/targets/%d/recordings/%d", target.Id, recordingId)) body := "SAVE" header := make(http.Header) header.Add("Content-Type", "text/plain") @@ -386,7 +386,7 @@ func (client *RecordingClient) Archive(ctx context.Context, target *Target, reco } func (client *RecordingClient) Stop(ctx context.Context, target *Target, recordingId uint32) error { - url := client.Base.JoinPath(fmt.Sprintf("/api/v3/targets/%d/recordings/%d", target.Id, recordingId)) + url := client.Base.JoinPath(fmt.Sprintf("/api/v4/targets/%d/recordings/%d", target.Id, recordingId)) body := "STOP" header := make(http.Header) header.Add("Content-Type", "text/plain") @@ -406,7 +406,7 @@ func (client *RecordingClient) Stop(ctx context.Context, target *Target, recordi } func (client *RecordingClient) Delete(ctx context.Context, target *Target, recordingId uint32) error { - url := client.Base.JoinPath(fmt.Sprintf("/api/v3/targets/%d/recordings/%d", target.Id, recordingId)) + url := client.Base.JoinPath(fmt.Sprintf("/api/v4/targets/%d/recordings/%d", target.Id, recordingId)) header := make(http.Header) resp, err := SendRequest(ctx, client.Client, http.MethodDelete, url.String(), nil, header) @@ -452,7 +452,7 @@ func (client *RecordingClient) GenerateReport(ctx context.Context, target *Targe } func (client *RecordingClient) ListArchives(ctx context.Context, target *Target) ([]Archive, error) { - url := client.Base.JoinPath("/api/v2.2/graphql") + url := client.Base.JoinPath("/api/v4/graphql") query := &GraphQLQuery{ Query: ` @@ -515,7 +515,7 @@ type CredentialClient struct { } func (client *CredentialClient) Create(ctx context.Context, credential *Credential) error { - url := client.Base.JoinPath("/api/v2.2/credentials") + url := client.Base.JoinPath("/api/v4/credentials") body := credential.ToFormData() header := make(http.Header) header.Add("Content-Type", "application/x-www-form-urlencoded") From fa2e9ccc008bd69d02287e5383120b1658f6bb03 Mon Sep 17 00:00:00 2001 From: Elliott Baron Date: Fri, 4 Oct 2024 14:56:28 -0400 Subject: [PATCH 6/6] fix scorecard tests for v4 API changes --- .../cryostat-operator.clusterserviceversion.yaml | 2 +- bundle/tests/scorecard/config.yaml | 12 ++++++------ config/scorecard/patches/custom.config.yaml | 12 ++++++------ internal/test/scorecard/clients.go | 4 ++-- internal/test/scorecard/types.go | 6 ------ 5 files changed, 15 insertions(+), 21 deletions(-) diff --git a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml index 326f8a19..5f2ab412 100644 --- a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml +++ b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml @@ -30,7 +30,7 @@ metadata: capabilities: Seamless Upgrades categories: Monitoring, Developer Tools containerImage: quay.io/cryostat/cryostat-operator:4.0.0-dev - createdAt: "2024-09-20T20:29:31Z" + createdAt: "2024-10-04T18:55:09Z" description: JVM monitoring and profiling tool operatorframework.io/initialization-resource: |- { diff --git a/bundle/tests/scorecard/config.yaml b/bundle/tests/scorecard/config.yaml index cc6e4cb4..73fd622e 100644 --- a/bundle/tests/scorecard/config.yaml +++ b/bundle/tests/scorecard/config.yaml @@ -70,7 +70,7 @@ stages: - entrypoint: - cryostat-scorecard-tests - operator-install - image: quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20240911172313 + image: quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20241004185220 labels: suite: cryostat test: operator-install @@ -80,7 +80,7 @@ stages: - entrypoint: - cryostat-scorecard-tests - cryostat-cr - image: quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20240911172313 + image: quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20241004185220 labels: suite: cryostat test: cryostat-cr @@ -90,7 +90,7 @@ stages: - entrypoint: - cryostat-scorecard-tests - cryostat-multi-namespace - image: quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20240911172313 + image: quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20241004185220 labels: suite: cryostat test: cryostat-multi-namespace @@ -100,7 +100,7 @@ stages: - entrypoint: - cryostat-scorecard-tests - cryostat-recording - image: quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20240911172313 + image: quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20241004185220 labels: suite: cryostat test: cryostat-recording @@ -110,7 +110,7 @@ stages: - entrypoint: - cryostat-scorecard-tests - cryostat-config-change - image: quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20240911172313 + image: quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20241004185220 labels: suite: cryostat test: cryostat-config-change @@ -120,7 +120,7 @@ stages: - entrypoint: - cryostat-scorecard-tests - cryostat-report - image: quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20240911172313 + image: quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20241004185220 labels: suite: cryostat test: cryostat-report diff --git a/config/scorecard/patches/custom.config.yaml b/config/scorecard/patches/custom.config.yaml index bec7ea86..b68184f2 100644 --- a/config/scorecard/patches/custom.config.yaml +++ b/config/scorecard/patches/custom.config.yaml @@ -8,7 +8,7 @@ entrypoint: - cryostat-scorecard-tests - operator-install - image: "quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20240911172313" + image: "quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20241004185220" labels: suite: cryostat test: operator-install @@ -18,7 +18,7 @@ entrypoint: - cryostat-scorecard-tests - cryostat-cr - image: "quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20240911172313" + image: "quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20241004185220" labels: suite: cryostat test: cryostat-cr @@ -28,7 +28,7 @@ entrypoint: - cryostat-scorecard-tests - cryostat-multi-namespace - image: "quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20240911172313" + image: "quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20241004185220" labels: suite: cryostat test: cryostat-multi-namespace @@ -38,7 +38,7 @@ entrypoint: - cryostat-scorecard-tests - cryostat-recording - image: "quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20240911172313" + image: "quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20241004185220" labels: suite: cryostat test: cryostat-recording @@ -48,7 +48,7 @@ entrypoint: - cryostat-scorecard-tests - cryostat-config-change - image: "quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20240911172313" + image: "quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20241004185220" labels: suite: cryostat test: cryostat-config-change @@ -58,7 +58,7 @@ entrypoint: - cryostat-scorecard-tests - cryostat-report - image: "quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20240911172313" + image: "quay.io/cryostat/cryostat-operator-scorecard:4.0.0-20241004185220" labels: suite: cryostat test: cryostat-report diff --git a/internal/test/scorecard/clients.go b/internal/test/scorecard/clients.go index 74b4382c..7fa5f83d 100644 --- a/internal/test/scorecard/clients.go +++ b/internal/test/scorecard/clients.go @@ -281,13 +281,13 @@ func (client *TargetClient) Create(ctx context.Context, options *Target) (*Targe return nil, fmt.Errorf("API request failed with status code: %d, response body: %s, and headers:\n%s", resp.StatusCode, ReadError(resp), ReadHeader(resp)) } - targetResp := &CustomTargetResponse{} + targetResp := &Target{} err = ReadJSON(resp, targetResp) if err != nil { return nil, fmt.Errorf("failed to read response body: %s", err.Error()) } - return targetResp.Data.Result, nil + return targetResp, nil } // Client for Cryostat Recording resources diff --git a/internal/test/scorecard/types.go b/internal/test/scorecard/types.go index 8c26603d..cdea8fa2 100644 --- a/internal/test/scorecard/types.go +++ b/internal/test/scorecard/types.go @@ -111,12 +111,6 @@ type Archive struct { Size uint32 `json:"size"` } -type CustomTargetResponse struct { - Data struct { - Result *Target `json:"result"` - } `json:"data"` -} - type Target struct { Id uint32 `json:"id,omitempty"` ConnectUrl string `json:"connectUrl"`