diff --git a/Makefile b/Makefile index ae45e0a0f..66c4b2941 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ CERTIFICATION_VERSION ?= $(VERSION) # The previous Operator version used to run the compatibility tests. COMPATIBLE_VERSION ?= 3.3.1 # The selector to use to find Operator Pods of the COMPATIBLE_VERSION (do not put in double quotes!!) -COMPATIBLE_SELECTOR = control-plane=coherence +COMPATIBLE_SELECTOR ?= control-plane=coherence # The GitHub project URL PROJECT_URL = https://github.com/oracle/coherence-operator @@ -1049,28 +1049,31 @@ run-prometheus-test: gotestsum # These tests will use whichever k8s cluster the local environment is pointing to. # ---------------------------------------------------------------------------------------------------------------------- .PHONY: compatibility-test -compatibility-test: export CGO_ENABLED = 0 -compatibility-test: export OPERATOR_NAMESPACE := $(OPERATOR_NAMESPACE) -compatibility-test: export CLUSTER_NAMESPACE := $(CLUSTER_NAMESPACE) -compatibility-test: export BUILD_OUTPUT := $(BUILD_OUTPUT) -compatibility-test: export TEST_APPLICATION_IMAGE := $(TEST_APPLICATION_IMAGE) -compatibility-test: export TEST_APPLICATION_IMAGE_CLIENT := $(TEST_APPLICATION_IMAGE_CLIENT) -compatibility-test: export TEST_APPLICATION_IMAGE_HELIDON := $(TEST_APPLICATION_IMAGE_HELIDON) -compatibility-test: export TEST_APPLICATION_IMAGE_SPRING := $(TEST_APPLICATION_IMAGE_SPRING) -compatibility-test: export TEST_APPLICATION_IMAGE_SPRING_FAT := $(TEST_APPLICATION_IMAGE_SPRING_FAT) -compatibility-test: export TEST_APPLICATION_IMAGE_SPRING_CNBP := $(TEST_APPLICATION_IMAGE_SPRING_CNBP) -compatibility-test: export TEST_COHERENCE_IMAGE := $(TEST_COHERENCE_IMAGE) -compatibility-test: export IMAGE_PULL_SECRETS := $(IMAGE_PULL_SECRETS) -compatibility-test: export TEST_SSL_SECRET := $(TEST_SSL_SECRET) -compatibility-test: export TEST_IMAGE_PULL_POLICY := $(IMAGE_PULL_POLICY) -compatibility-test: export TEST_STORAGE_CLASS := $(TEST_STORAGE_CLASS) -compatibility-test: export VERSION := $(VERSION) -compatibility-test: export COMPATIBLE_VERSION := $(COMPATIBLE_VERSION) -compatibility-test: export COMPATIBLE_SELECTOR := $(COMPATIBLE_SELECTOR) -compatibility-test: export OPERATOR_IMAGE := $(OPERATOR_IMAGE) -compatibility-test: export COHERENCE_IMAGE := $(COHERENCE_IMAGE) -compatibility-test: export GO_TEST_FLAGS_E2E := $(strip $(GO_TEST_FLAGS_E2E)) -compatibility-test: undeploy build-all-images $(BUILD_HELM)/coherence-operator-$(VERSION).tgz undeploy clean-namespace reset-namespace gotestsum ## Run the Operator backwards compatibility tests +compatibility-test: undeploy build-all-images $(BUILD_HELM)/coherence-operator-$(VERSION).tgz undeploy clean-namespace reset-namespace gotestsum just-compatibility-test ## Run the Operator backwards compatibility tests + +.PHONY: just-compatibility-test +just-compatibility-test: export CGO_ENABLED = 0 +just-compatibility-test: export OPERATOR_NAMESPACE := $(OPERATOR_NAMESPACE) +just-compatibility-test: export CLUSTER_NAMESPACE := $(CLUSTER_NAMESPACE) +just-compatibility-test: export BUILD_OUTPUT := $(BUILD_OUTPUT) +just-compatibility-test: export TEST_APPLICATION_IMAGE := $(TEST_APPLICATION_IMAGE) +just-compatibility-test: export TEST_APPLICATION_IMAGE_CLIENT := $(TEST_APPLICATION_IMAGE_CLIENT) +just-compatibility-test: export TEST_APPLICATION_IMAGE_HELIDON := $(TEST_APPLICATION_IMAGE_HELIDON) +just-compatibility-test: export TEST_APPLICATION_IMAGE_SPRING := $(TEST_APPLICATION_IMAGE_SPRING) +just-compatibility-test: export TEST_APPLICATION_IMAGE_SPRING_FAT := $(TEST_APPLICATION_IMAGE_SPRING_FAT) +just-compatibility-test: export TEST_APPLICATION_IMAGE_SPRING_CNBP := $(TEST_APPLICATION_IMAGE_SPRING_CNBP) +just-compatibility-test: export TEST_COHERENCE_IMAGE := $(TEST_COHERENCE_IMAGE) +just-compatibility-test: export IMAGE_PULL_SECRETS := $(IMAGE_PULL_SECRETS) +just-compatibility-test: export TEST_SSL_SECRET := $(TEST_SSL_SECRET) +just-compatibility-test: export TEST_IMAGE_PULL_POLICY := $(IMAGE_PULL_POLICY) +just-compatibility-test: export TEST_STORAGE_CLASS := $(TEST_STORAGE_CLASS) +just-compatibility-test: export VERSION := $(VERSION) +just-compatibility-test: export COMPATIBLE_VERSION := $(COMPATIBLE_VERSION) +just-compatibility-test: export COMPATIBLE_SELECTOR := $(COMPATIBLE_SELECTOR) +just-compatibility-test: export OPERATOR_IMAGE := $(OPERATOR_IMAGE) +just-compatibility-test: export COHERENCE_IMAGE := $(COHERENCE_IMAGE) +just-compatibility-test: export GO_TEST_FLAGS_E2E := $(strip $(GO_TEST_FLAGS_E2E)) +just-compatibility-test: ## Run the Operator backwards compatibility tests WITHOUT building anything helm repo add coherence https://oracle.github.io/coherence-operator/charts helm repo update $(GOTESTSUM) --format standard-verbose --junitfile $(TEST_LOGS_DIR)/operator-e2e-compatibility-test.xml \ @@ -2107,6 +2110,9 @@ install-istio: get-istio ## Install the latest version of Istio into k8s (or ove kubectl apply -f ./hack/istio-strict.yaml kubectl -n $(OPERATOR_NAMESPACE) apply -f ./hack/istio-operator.yaml kubectl label namespace $(OPERATOR_NAMESPACE) istio-injection=enabled --overwrite=true + kubectl label namespace $(OPERATOR_NAMESPACE_CLIENT) istio-injection=enabled --overwrite=true + kubectl label namespace $(CLUSTER_NAMESPACE) istio-injection=enabled --overwrite=true + kubectl apply -f $(ISTIO_HOME)/samples/addons # ---------------------------------------------------------------------------------------------------------------------- # Uninstall Istio diff --git a/api/v1/coherence_types.go b/api/v1/coherence_types.go index fd7d66a0f..001f23cf0 100644 --- a/api/v1/coherence_types.go +++ b/api/v1/coherence_types.go @@ -1065,6 +1065,14 @@ type NamedPortSpec struct { // port. // +optional ServiceMonitor *ServiceMonitorSpec `json:"serviceMonitor,omitempty"` + // ExposeOnSTS is a flag to indicate that this port should also be exposed on + // the StatefulSetHeadless service. This is useful in cases where a service mesh + // such as Istio is being used and ports such as the Extend or gRPC ports are + // accessed via the StatefulSet service. + // The default is `true` so all additional ports are exposed on the StatefulSet + // headless service. + // +optional + ExposeOnSTS *bool `json:"exposeOnSts,omitempty"` } // GetServiceName returns the name of the Service used to expose this port, or returns @@ -1090,13 +1098,6 @@ func (in *NamedPortSpec) CreateService(deployment CoherenceResource) *corev1.Ser name, _ := in.GetServiceName(deployment) - var portName string - if in.Service != nil && in.Service.PortName != nil { - portName = *in.Service.PortName - } else { - portName = in.Name - } - // The labels for the service svcLabels := deployment.CreateCommonLabels() svcLabels[LabelComponent] = LabelComponentPortService @@ -1118,17 +1119,7 @@ func (in *NamedPortSpec) CreateService(deployment CoherenceResource) *corev1.Ser // Add the port serviceSpec.Ports = []corev1.ServicePort{ - { - Name: portName, - Protocol: in.GetProtocol(), - Port: in.GetServicePort(deployment), - TargetPort: intstr.FromInt(int(in.GetPort(deployment))), - NodePort: in.GetNodePort(), - }, - } - - if in.AppProtocol != nil { - serviceSpec.Ports[0].AppProtocol = in.AppProtocol + in.createServicePort(deployment), } // Add the service selector @@ -1148,6 +1139,31 @@ func (in *NamedPortSpec) CreateService(deployment CoherenceResource) *corev1.Ser return &svc } +func (in *NamedPortSpec) createServicePort(deployment CoherenceResource) corev1.ServicePort { + var portName string + if in.Service != nil && in.Service.PortName != nil { + portName = *in.Service.PortName + } else { + portName = in.Name + } + + sp := corev1.ServicePort{ + Name: portName, + Protocol: in.GetProtocol(), + Port: in.GetServicePort(deployment), + TargetPort: intstr.FromInt32(in.GetPort(deployment)), + NodePort: in.GetNodePort(), + } + + if in.AppProtocol != nil { + sp.AppProtocol = in.AppProtocol + } else { + sp.AppProtocol = in.GetDefaultAppProtocol() + } + + return sp +} + // CreateServiceMonitor creates the Prometheus ServiceMonitor to expose this port if enabled. func (in *NamedPortSpec) CreateServiceMonitor(deployment CoherenceResource) *monitoringv1.ServiceMonitor { if in == nil || !in.IsEnabled() { @@ -1245,6 +1261,21 @@ func (in *NamedPortSpec) GetServicePort(d CoherenceResource) int32 { } } +func (in *NamedPortSpec) GetDefaultAppProtocol() *string { + switch { + case in == nil: + return nil + case strings.ToLower(in.Name) == PortNameMetrics: + // special case for well known port - metrics + return pointer.String(AppProtocolHttp) + case in.Port == 0 && strings.ToLower(in.Name) == PortNameManagement: + // special case for well known port - management + return pointer.String(AppProtocolHttp) + default: + return nil + } +} + func (in *NamedPortSpec) GetNodePort() int32 { if in == nil || in.NodePort == nil { return 0 @@ -2142,7 +2173,7 @@ func (in *ReadinessProbeSpec) UpdateProbeSpec(port int32, path string, probe *co default: probe.HTTPGet = &corev1.HTTPGetAction{ Path: path, - Port: intstr.FromInt(int(port)), + Port: intstr.FromInt32(port), Scheme: corev1.URISchemeHTTP, } } diff --git a/api/v1/coherence_webhook_job_test.go b/api/v1/coherence_webhook_job_test.go index 3074b956c..ad3e3ce1f 100644 --- a/api/v1/coherence_webhook_job_test.go +++ b/api/v1/coherence_webhook_job_test.go @@ -135,7 +135,7 @@ func TestJobCoherenceLocalPortIsNotSetOnUpdate(t *testing.T) { func TestJobCoherenceLocalPortAdjustIsSet(t *testing.T) { g := NewGomegaWithT(t) - lpa := intstr.FromInt(int(coh.DefaultUnicastPortAdjust)) + lpa := intstr.FromInt32(coh.DefaultUnicastPortAdjust) c := coh.CoherenceJob{} c.Default() g.Expect(c.Spec.Coherence).NotTo(BeNil()) @@ -145,7 +145,7 @@ func TestJobCoherenceLocalPortAdjustIsSet(t *testing.T) { func TestJobCoherenceLocalPortAdjustIsNotOverridden(t *testing.T) { g := NewGomegaWithT(t) - lpa := intstr.FromInt(9876) + lpa := intstr.FromInt32(9876) c := coh.CoherenceJob{ Spec: coh.CoherenceJobResourceSpec{ CoherenceResourceSpec: coh.CoherenceResourceSpec{ diff --git a/api/v1/coherence_webhook_test.go b/api/v1/coherence_webhook_test.go index 78c2197c6..83611800c 100644 --- a/api/v1/coherence_webhook_test.go +++ b/api/v1/coherence_webhook_test.go @@ -134,7 +134,7 @@ func TestCoherenceLocalPortIsNotSetOnUpdate(t *testing.T) { func TestCoherenceLocalPortAdjustIsSet(t *testing.T) { g := NewGomegaWithT(t) - lpa := intstr.FromInt(int(coh.DefaultUnicastPortAdjust)) + lpa := intstr.FromInt32(coh.DefaultUnicastPortAdjust) c := coh.Coherence{} c.Default() g.Expect(c.Spec.Coherence).NotTo(BeNil()) @@ -144,7 +144,7 @@ func TestCoherenceLocalPortAdjustIsSet(t *testing.T) { func TestCoherenceLocalPortAdjustIsNotOverridden(t *testing.T) { g := NewGomegaWithT(t) - lpa := intstr.FromInt(9876) + lpa := intstr.FromInt32(9876) c := coh.Coherence{ Spec: coh.CoherenceStatefulSetResourceSpec{ CoherenceResourceSpec: coh.CoherenceResourceSpec{ diff --git a/api/v1/coherencejobresource_types.go b/api/v1/coherencejobresource_types.go index 9bf003fc5..0e04bbbc3 100644 --- a/api/v1/coherencejobresource_types.go +++ b/api/v1/coherencejobresource_types.go @@ -47,7 +47,6 @@ func (in *CoherenceJob) GetCoherenceClusterName() string { if in == nil { return "" } - if in.Spec.Cluster == "" { return in.Name } @@ -93,6 +92,17 @@ func (in *CoherenceJob) AddAnnotation(key, value string) { } } +func (in *CoherenceJob) AddAnnotationIfMissing(key, value string) { + if in != nil { + if in.Annotations == nil { + in.Annotations = make(map[string]string) + } + if _, found := in.Annotations[key]; !found { + in.Annotations[key] = value + } + } +} + // GetStatus returns this resource's CoherenceResourceSpec func (in *CoherenceJob) GetStatus() *CoherenceResourceStatus { return &in.Status diff --git a/api/v1/coherenceresource.go b/api/v1/coherenceresource.go index 61e020e33..988aab094 100644 --- a/api/v1/coherenceresource.go +++ b/api/v1/coherenceresource.go @@ -76,6 +76,8 @@ type CoherenceResource interface { GetStatus() *CoherenceResourceStatus // AddAnnotation adds an annotation to this resource AddAnnotation(key, value string) + // AddAnnotationIfMissing adds an annotation to this resource if it is not already present + AddAnnotationIfMissing(key, value string) // GetAnnotations returns the annotations on this resource GetAnnotations() map[string]string // CreateKubernetesResources creates the kubernetes resources defined by this resource diff --git a/api/v1/coherenceresource_types.go b/api/v1/coherenceresource_types.go index 78960924f..09b137c35 100644 --- a/api/v1/coherenceresource_types.go +++ b/api/v1/coherenceresource_types.go @@ -271,6 +271,17 @@ func (in *Coherence) AddAnnotation(key, value string) { } } +func (in *Coherence) AddAnnotationIfMissing(key, value string) { + if in != nil { + if in.Annotations == nil { + in.Annotations = make(map[string]string) + } + if _, found := in.Annotations[key]; !found { + in.Annotations[key] = value + } + } +} + // GetNamespacedName returns the namespace/name key to look up this resource. func (in *Coherence) GetNamespacedName() types.NamespacedName { return types.NamespacedName{ @@ -321,7 +332,14 @@ func (in *Coherence) GetVersionAnnotation() (string, bool) { // before the specified version, or is not set. // The version parameter must be a valid SemVer value. func (in *Coherence) IsBeforeVersion(version string) bool { + if version[0] != 'v' { + version = "v" + version + } + if actual, found := in.GetVersionAnnotation(); found { + if actual[0] != 'v' { + actual = "v" + actual + } return semver.Compare(actual, version) < 0 } return true diff --git a/api/v1/coherenceresourcespec_types.go b/api/v1/coherenceresourcespec_types.go index 5570fa21b..efd420ff5 100644 --- a/api/v1/coherenceresourcespec_types.go +++ b/api/v1/coherenceresourcespec_types.go @@ -530,15 +530,8 @@ func (in *CoherenceResourceSpec) CreateWKAService(deployment CoherenceResource) ClusterIP: corev1.ClusterIPNone, // Pods must be part of the WKA service even if not ready PublishNotReadyAddresses: true, - Ports: []corev1.ServicePort{ - { - Name: "tcp-" + PortNameCoherence, - Protocol: corev1.ProtocolTCP, - Port: 7, - TargetPort: intstr.FromInt(7), - }, - }, - Selector: selector, + Ports: in.createDefaultServicePorts(), + Selector: selector, }, } @@ -558,6 +551,13 @@ func (in *CoherenceResourceSpec) CreateHeadlessService(deployment CoherenceResou // The selector for the service selector := in.CreatePodSelectorLabels(deployment) + ports := in.createDefaultServicePorts() + for _, p := range in.Ports { + if p.ExposeOnSTS == nil || *p.ExposeOnSTS { + ports = append(ports, p.createServicePort(deployment)) + } + } + // Create the Service svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -569,14 +569,7 @@ func (in *CoherenceResourceSpec) CreateHeadlessService(deployment CoherenceResou ClusterIP: "None", PublishNotReadyAddresses: true, Selector: selector, - Ports: []corev1.ServicePort{ - { - Name: "tcp-" + PortNameCoherence, - Protocol: corev1.ProtocolTCP, - Port: 7, - TargetPort: intstr.FromInt(7), - }, - }, + Ports: ports, }, } @@ -587,6 +580,42 @@ func (in *CoherenceResourceSpec) CreateHeadlessService(deployment CoherenceResou } } +func (in *CoherenceResourceSpec) createDefaultServicePorts() []corev1.ServicePort { + hp := in.GetHealthPort() + lp, _ := in.Coherence.GetLocalPorts() + + return []corev1.ServicePort{ + { + Name: PortNameCoherence, + Protocol: corev1.ProtocolTCP, + Port: 7, + TargetPort: intstr.FromInt32(7), + AppProtocol: pointer.String(AppProtocolTcp), + }, + { + Name: PortNameCoherenceLocal, + Protocol: corev1.ProtocolTCP, + AppProtocol: pointer.String(AppProtocolTcp), + Port: lp, + TargetPort: intstr.FromString(PortNameCoherenceLocal), + }, + { + Name: PortNameCoherenceCluster, + Protocol: corev1.ProtocolTCP, + AppProtocol: pointer.String(AppProtocolTcp), + Port: DefaultClusterPort, + TargetPort: intstr.FromString(PortNameCoherenceCluster), + }, + { + Name: PortNameHealth, + Protocol: corev1.ProtocolTCP, + AppProtocol: pointer.String(AppProtocolHttp), + Port: hp, + TargetPort: intstr.FromString(PortNameHealth), + }, + } +} + func (in *CoherenceResourceSpec) CreatePodTemplateSpec(deployment CoherenceResource) corev1.PodTemplateSpec { // Create the PodSpec labels podLabels := in.CreatePodSelectorLabels(deployment) @@ -703,11 +732,19 @@ func (in *CoherenceResourceSpec) CreateCoherenceContainer(deployment CoherenceRe healthPort := in.GetHealthPort() vm := in.CreateCommonVolumeMounts() + lp, _ := in.Coherence.GetLocalPorts() + + cmd := []string{RunnerCommand} + if in.Application != nil && in.Application.Type != nil && *in.Application.Type == "operator" { + cmd = append(cmd, in.Application.Args...) + } else { + cmd = append(cmd, "server") + } c := corev1.Container{ Name: ContainerNameCoherence, Image: cohImage, - Command: []string{RunnerCommand, "server"}, + Command: cmd, Env: in.Env, Ports: []corev1.ContainerPort{ { @@ -720,6 +757,16 @@ func (in *CoherenceResourceSpec) CreateCoherenceContainer(deployment CoherenceRe ContainerPort: healthPort, Protocol: corev1.ProtocolTCP, }, + { + Name: PortNameCoherenceLocal, + ContainerPort: lp, + Protocol: corev1.ProtocolTCP, + }, + { + Name: PortNameCoherenceCluster, + ContainerPort: DefaultClusterPort, + Protocol: corev1.ProtocolTCP, + }, }, SecurityContext: in.ContainerSecurityContext, VolumeMounts: vm, @@ -772,7 +819,7 @@ func (in *CoherenceResourceSpec) CreateCommonVolumeMounts() []corev1.VolumeMount // CreateCommonEnv creates the common environment variables added all. func (in *CoherenceResourceSpec) CreateCommonEnv(deployment CoherenceResource) []corev1.EnvVar { - return []corev1.EnvVar{ + env := []corev1.EnvVar{ { Name: EnvVarCohMachineName, ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ @@ -794,9 +841,15 @@ func (in *CoherenceResourceSpec) CreateCommonEnv(deployment CoherenceResource) [ }, }, }, - {Name: EnvVarCohClusterName, Value: deployment.GetCoherenceClusterName()}, {Name: EnvVarCohRole, Value: deployment.GetRoleName()}, } + + clusterName := deployment.GetCoherenceClusterName() + if clusterName != "" { + env = append(env, corev1.EnvVar{Name: EnvVarCohClusterName, Value: clusterName}) + } + + return env } // AddEnvVarIfAbsent adds the specified EnvVar if one with the same name does not already exist. @@ -895,7 +948,7 @@ func (in *CoherenceResourceSpec) CreateDefaultReadinessProbe() *corev1.Probe { func (in *CoherenceResourceSpec) UpdateDefaultReadinessProbeAction(probe *corev1.Probe) *corev1.Probe { probe.HTTPGet = &corev1.HTTPGetAction{ Path: DefaultReadinessPath, - Port: intstr.FromInt(int(DefaultHealthPort)), + Port: intstr.FromInt32(DefaultHealthPort), Scheme: corev1.URISchemeHTTP, } return probe @@ -916,7 +969,7 @@ func (in *CoherenceResourceSpec) CreateDefaultLivenessProbe() *corev1.Probe { func (in *CoherenceResourceSpec) UpdateDefaultLivenessProbeAction(probe *corev1.Probe) *corev1.Probe { probe.HTTPGet = &corev1.HTTPGetAction{ Path: DefaultLivenessPath, - Port: intstr.FromInt(int(DefaultHealthPort)), + Port: intstr.FromInt32(DefaultHealthPort), Scheme: corev1.URISchemeHTTP, } return probe @@ -933,14 +986,20 @@ func (in *CoherenceResourceSpec) CreateOperatorInitContainer(deployment Coherenc vm := in.CreateCommonVolumeMounts() + env := []corev1.EnvVar{ + {Name: EnvVarCohUtilDir, Value: VolumeMountPathUtils}, + } + + clusterName := deployment.GetCoherenceClusterName() + if clusterName != "" { + env = append(env, corev1.EnvVar{Name: EnvVarCohClusterName, Value: clusterName}) + } + c := corev1.Container{ - Name: ContainerNameOperatorInit, - Image: image, - Command: []string{RunnerInitCommand, RunnerInit}, - Env: []corev1.EnvVar{ - {Name: EnvVarCohUtilDir, Value: VolumeMountPathUtils}, - {Name: EnvVarCohClusterName, Value: deployment.GetCoherenceClusterName()}, - }, + Name: ContainerNameOperatorInit, + Image: image, + Command: []string{RunnerInitCommand, RunnerInit}, + Env: env, SecurityContext: in.ContainerSecurityContext, VolumeMounts: vm, } diff --git a/api/v1/common_test.go b/api/v1/common_test.go index 2df307548..b05cf5329 100644 --- a/api/v1/common_test.go +++ b/api/v1/common_test.go @@ -367,6 +367,8 @@ func createMinimalExpectedPodSpec(deployment coh.CoherenceResource) corev1.PodTe }) } + lp, _ := deployment.GetSpec().Coherence.GetLocalPorts() + // The Coherence Container cohContainer := corev1.Container{ Name: coh.ContainerNameCoherence, @@ -374,15 +376,25 @@ func createMinimalExpectedPodSpec(deployment coh.CoherenceResource) corev1.PodTe Command: []string{coh.RunnerCommand, "server"}, Ports: []corev1.ContainerPort{ { - Name: "coherence", + Name: coh.PortNameCoherence, ContainerPort: 7, Protocol: "TCP", }, { - Name: "health", + Name: coh.PortNameHealth, ContainerPort: spec.GetHealthPort(), Protocol: "TCP", }, + { + Name: coh.PortNameCoherenceLocal, + ContainerPort: lp, + Protocol: corev1.ProtocolTCP, + }, + { + Name: coh.PortNameCoherenceCluster, + ContainerPort: coh.DefaultClusterPort, + Protocol: corev1.ProtocolTCP, + }, }, ReadinessProbe: spec.UpdateDefaultReadinessProbeAction(spec.CreateDefaultReadinessProbe()), LivenessProbe: spec.UpdateDefaultLivenessProbeAction(spec.CreateDefaultLivenessProbe()), diff --git a/api/v1/constants.go b/api/v1/constants.go index 2471b1e4c..074793c58 100644 --- a/api/v1/constants.go +++ b/api/v1/constants.go @@ -123,6 +123,10 @@ const ( // PortNameCoherence is the name of the Coherence port PortNameCoherence = "coherence" + // PortNameCoherenceLocal is the name of the Coherence local port + PortNameCoherenceLocal = "coh-local" + // PortNameCoherenceCluster is the name of the Coherence cluster port + PortNameCoherenceCluster = "coh-cluster" // PortNameDebug is the name of the debug port PortNameDebug = "debug-port" // PortNameHealth is the name of the health port @@ -132,6 +136,11 @@ const ( // PortNameMetrics is the name of the Coherence metrics port PortNameMetrics = "metrics" + // AppProtocolTcp is the appProtocol value for ports that use tcp + AppProtocolTcp = "tcp" + // AppProtocolHttp is the appProtocol value for ports that use http + AppProtocolHttp = "http" + // DefaultDebugPort is the default debug port DefaultDebugPort int32 = 5005 // DefaultManagementPort is the Coherence manaement debug port @@ -140,6 +149,8 @@ const ( DefaultMetricsPort int32 = 9612 // DefaultHealthPort is the default health port DefaultHealthPort int32 = 6676 + // DefaultClusterPort is the default Coherence cluster port + DefaultClusterPort int32 = 7574 // DefaultUnicastPort is the default Coherence unicast port DefaultUnicastPort int32 = 7575 // DefaultUnicastPortAdjust is the default Coherence unicast port adjust value diff --git a/api/v1/create_job_coherencespec_test.go b/api/v1/create_job_coherencespec_test.go index 646bfc131..49e0e8dc1 100644 --- a/api/v1/create_job_coherencespec_test.go +++ b/api/v1/create_job_coherencespec_test.go @@ -135,7 +135,7 @@ func TestCreateJobWithCoherenceLocalPortAdjustFalse(t *testing.T) { } func TestCreateJobWithCoherenceLocalPortAdjust(t *testing.T) { - lpa := intstr.FromInt(9876) + lpa := intstr.FromInt32(9876) spec := coh.CoherenceResourceSpec{ Coherence: &coh.CoherenceSpec{ LocalPortAdjust: &lpa, diff --git a/api/v1/create_job_probes_test.go b/api/v1/create_job_probes_test.go index 8b246654c..8290ed66e 100644 --- a/api/v1/create_job_probes_test.go +++ b/api/v1/create_job_probes_test.go @@ -55,7 +55,7 @@ func TestCreateJobWithReadinessProbeSpec(t *testing.T) { Exec: nil, HTTPGet: &corev1.HTTPGetAction{ Path: coh.DefaultReadinessPath, - Port: intstr.FromInt(int(coh.DefaultHealthPort)), + Port: intstr.FromInt32(coh.DefaultHealthPort), Scheme: "HTTP", }, TCPSocket: nil, @@ -74,7 +74,7 @@ func TestCreateJobWithReadinessProbeSpec(t *testing.T) { func TestCreateJobWithReadinessProbeSpecWithHttpGet(t *testing.T) { handler := &corev1.HTTPGetAction{ Path: "/test/ready", - Port: intstr.FromInt(1234), + Port: intstr.FromInt32(1234), } probe := coh.ReadinessProbeSpec{ @@ -115,7 +115,7 @@ func TestCreateJobWithReadinessProbeSpecWithHttpGet(t *testing.T) { func TestCreateJobWithReadinessProbeSpecWithTCPSocket(t *testing.T) { handler := &corev1.TCPSocketAction{ - Port: intstr.FromInt(1234), + Port: intstr.FromInt32(1234), Host: "foo.com", } @@ -235,7 +235,7 @@ func TestCreateJobWithLivenessProbeSpec(t *testing.T) { Exec: nil, HTTPGet: &corev1.HTTPGetAction{ Path: coh.DefaultLivenessPath, - Port: intstr.FromInt(int(coh.DefaultHealthPort)), + Port: intstr.FromInt32(coh.DefaultHealthPort), Scheme: "HTTP", }, TCPSocket: nil, @@ -255,7 +255,7 @@ func TestCreateJobWithLivenessProbeSpecWithHttpGet(t *testing.T) { handler := &corev1.HTTPGetAction{ Path: "/test/ready", - Port: intstr.FromInt(1234), + Port: intstr.FromInt32(1234), } probe := coh.ReadinessProbeSpec{ @@ -297,7 +297,7 @@ func TestCreateJobWithLivenessProbeSpecWithHttpGet(t *testing.T) { func TestCreateJobWithLivenessProbeSpecWithTCPSocket(t *testing.T) { handler := &corev1.TCPSocketAction{ - Port: intstr.FromInt(1234), + Port: intstr.FromInt32(1234), Host: "foo.com", } @@ -436,7 +436,7 @@ func TestCreateJobWithStartupProbeSpec(t *testing.T) { Exec: nil, HTTPGet: &corev1.HTTPGetAction{ Path: coh.DefaultLivenessPath, - Port: intstr.FromInt(int(coh.DefaultHealthPort)), + Port: intstr.FromInt32(coh.DefaultHealthPort), Scheme: "HTTP", }, TCPSocket: nil, @@ -456,7 +456,7 @@ func TestCreateJobWithStartupProbeSpecWithHttpGet(t *testing.T) { handler := &corev1.HTTPGetAction{ Path: "/test/ready", - Port: intstr.FromInt(1234), + Port: intstr.FromInt32(1234), } probe := coh.ReadinessProbeSpec{ @@ -498,7 +498,7 @@ func TestCreateJobWithStartupProbeSpecWithHttpGet(t *testing.T) { func TestCreateJobWithStartupProbeSpecWithTCPSocket(t *testing.T) { handler := &corev1.TCPSocketAction{ - Port: intstr.FromInt(1234), + Port: intstr.FromInt32(1234), Host: "foo.com", } diff --git a/api/v1/create_job_wka_services_test.go b/api/v1/create_job_wka_services_test.go index a593c6337..e74d96f84 100644 --- a/api/v1/create_job_wka_services_test.go +++ b/api/v1/create_job_wka_services_test.go @@ -12,7 +12,6 @@ import ( coh "github.com/oracle/coherence-operator/api/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" "testing" ) @@ -49,15 +48,8 @@ func TestCreateWKAServiceForMinimalJonDeployment(t *testing.T) { ClusterIP: corev1.ClusterIPNone, // Pods must be part of the WKA service even if not ready PublishNotReadyAddresses: true, - Ports: []corev1.ServicePort{ - { - Name: "tcp-" + coh.PortNameCoherence, - Protocol: corev1.ProtocolTCP, - Port: 7, - TargetPort: intstr.FromInt(7), - }, - }, - Selector: selector, + Ports: getDefaultServicePorts(), + Selector: selector, }, } @@ -104,15 +96,8 @@ func TestCreateWKAServiceForJobWithAppLabel(t *testing.T) { ClusterIP: corev1.ClusterIPNone, // Pods must be part of the WKA service even if not ready PublishNotReadyAddresses: true, - Ports: []corev1.ServicePort{ - { - Name: "tcp-" + coh.PortNameCoherence, - Protocol: corev1.ProtocolTCP, - Port: 7, - TargetPort: intstr.FromInt(7), - }, - }, - Selector: selector, + Ports: getDefaultServicePorts(), + Selector: selector, }, } @@ -159,15 +144,8 @@ func TestCreateWKAServiceForJobWithVersionLabel(t *testing.T) { ClusterIP: corev1.ClusterIPNone, // Pods must be part of the WKA service even if not ready PublishNotReadyAddresses: true, - Ports: []corev1.ServicePort{ - { - Name: "tcp-" + coh.PortNameCoherence, - Protocol: corev1.ProtocolTCP, - Port: 7, - TargetPort: intstr.FromInt(7), - }, - }, - Selector: selector, + Ports: getDefaultServicePorts(), + Selector: selector, }, } @@ -211,15 +189,8 @@ func TestCreateWKAServiceForJobWithClusterName(t *testing.T) { ClusterIP: corev1.ClusterIPNone, // Pods must be part of the WKA service even if not ready PublishNotReadyAddresses: true, - Ports: []corev1.ServicePort{ - { - Name: "tcp-" + coh.PortNameCoherence, - Protocol: corev1.ProtocolTCP, - Port: 7, - TargetPort: intstr.FromInt(7), - }, - }, - Selector: selector, + Ports: getDefaultServicePorts(), + Selector: selector, }, } diff --git a/api/v1/create_port_service_test.go b/api/v1/create_port_service_test.go index 6280fb1b5..307512cf3 100644 --- a/api/v1/create_port_service_test.go +++ b/api/v1/create_port_service_test.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2021, Oracle and/or its affiliates. + * Copyright (c) 2019, 2023, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -45,7 +45,7 @@ func TestNamedPortSpec_CreateServiceWithMinimalFields(t *testing.T) { Name: "foo", Protocol: corev1.ProtocolTCP, Port: 19, - TargetPort: intstr.FromInt(19), + TargetPort: intstr.FromInt32(19), NodePort: 0, }, }, @@ -236,7 +236,7 @@ func TestNamedPortSpec_CreateServiceWithService(t *testing.T) { Name: "foo", Protocol: corev1.ProtocolTCP, Port: 99, - TargetPort: intstr.FromInt(19), + TargetPort: intstr.FromInt32(19), }, }, Selector: selector, diff --git a/api/v1/create_services_for_ports_test.go b/api/v1/create_services_for_ports_test.go index 18fd4fc7d..4ef377ae9 100644 --- a/api/v1/create_services_for_ports_test.go +++ b/api/v1/create_services_for_ports_test.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2021, Oracle and/or its affiliates. + * Copyright (c) 2020, 2023, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -103,7 +103,7 @@ func TestCreateServicesWithPortsWithOneAdditionalPort(t *testing.T) { Name: "test-port-one", Protocol: protocol, Port: 9876, - TargetPort: intstr.FromInt(9876), + TargetPort: intstr.FromInt32(9876), NodePort: 2020, }, }, @@ -160,7 +160,7 @@ func TestCreateServicesWithPortsWithOneAdditionalPortWithServiceName(t *testing. Name: "test-port-one", Protocol: protocol, Port: 9876, - TargetPort: intstr.FromInt(9876), + TargetPort: intstr.FromInt32(9876), NodePort: 2020, }, }, @@ -217,7 +217,7 @@ func TestCreateServicesWithPortsWithOneAdditionalPortWithServicePort(t *testing. Name: "test-port-one", Protocol: protocol, Port: 80, - TargetPort: intstr.FromInt(9876), + TargetPort: intstr.FromInt32(9876), NodePort: 2020, }, }, @@ -296,7 +296,7 @@ func TestCreateServicesWithPortsWithOneAdditionalPortWithServiceFields(t *testin Name: "test-port-one", Protocol: protocol, Port: 9876, - TargetPort: intstr.FromInt(9876), + TargetPort: intstr.FromInt32(9876), NodePort: 2020, }, }, @@ -361,7 +361,7 @@ func TestCreateServicesWithPortsWithOneAdditionalPortWithServiceLabels(t *testin { Name: "test-port-one", Port: 9876, - TargetPort: intstr.FromInt(9876), + TargetPort: intstr.FromInt32(9876), Protocol: corev1.ProtocolTCP, }, }, @@ -412,7 +412,7 @@ func TestCreateServicesWithPortsWithOneAdditionalPortWithServiceAnnotations(t *t { Name: "test-port-one", Port: 9876, - TargetPort: intstr.FromInt(9876), + TargetPort: intstr.FromInt32(9876), Protocol: corev1.ProtocolTCP, }, }, @@ -472,7 +472,7 @@ func TestCreateServicesWithPortsWithTwoAdditionalPorts(t *testing.T) { { Name: "test-port-one", Port: 9876, - TargetPort: intstr.FromInt(9876), + TargetPort: intstr.FromInt32(9876), Protocol: protocolOne, }, }, @@ -491,7 +491,7 @@ func TestCreateServicesWithPortsWithTwoAdditionalPorts(t *testing.T) { { Name: "test-port-two", Port: 5678, - TargetPort: intstr.FromInt(5678), + TargetPort: intstr.FromInt32(5678), Protocol: protocolTwo, }, }, diff --git a/api/v1/create_statefulset_coherencespec_test.go b/api/v1/create_statefulset_coherencespec_test.go index fe40d4810..4001ebb01 100644 --- a/api/v1/create_statefulset_coherencespec_test.go +++ b/api/v1/create_statefulset_coherencespec_test.go @@ -135,7 +135,7 @@ func TestCreateStatefulSetWithCoherenceLocalPortAdjustFalse(t *testing.T) { } func TestCreateStatefulSetWithCoherenceLocalPortAdjust(t *testing.T) { - lpa := intstr.FromInt(9876) + lpa := intstr.FromInt32(9876) spec := coh.CoherenceResourceSpec{ Coherence: &coh.CoherenceSpec{ LocalPortAdjust: &lpa, diff --git a/api/v1/create_statefulset_probes_test.go b/api/v1/create_statefulset_probes_test.go index f6cfc979f..f9a0ee9aa 100644 --- a/api/v1/create_statefulset_probes_test.go +++ b/api/v1/create_statefulset_probes_test.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Oracle and/or its affiliates. + * Copyright (c) 2022, 2023, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -57,7 +57,7 @@ func TestCreateStatefulSetWithReadinessProbeSpec(t *testing.T) { Exec: nil, HTTPGet: &corev1.HTTPGetAction{ Path: coh.DefaultReadinessPath, - Port: intstr.FromInt(int(coh.DefaultHealthPort)), + Port: intstr.FromInt32(coh.DefaultHealthPort), Scheme: "HTTP", }, TCPSocket: nil, @@ -77,7 +77,7 @@ func TestCreateStatefulSetWithReadinessProbeSpecWithHttpGet(t *testing.T) { handler := &corev1.HTTPGetAction{ Path: "/test/ready", - Port: intstr.FromInt(1234), + Port: intstr.FromInt32(1234), } probe := coh.ReadinessProbeSpec{ @@ -119,7 +119,7 @@ func TestCreateStatefulSetWithReadinessProbeSpecWithHttpGet(t *testing.T) { func TestCreateStatefulSetWithReadinessProbeSpecWithTCPSocket(t *testing.T) { handler := &corev1.TCPSocketAction{ - Port: intstr.FromInt(1234), + Port: intstr.FromInt32(1234), Host: "foo.com", } @@ -241,7 +241,7 @@ func TestCreateStatefulSetWithLivenessProbeSpec(t *testing.T) { Exec: nil, HTTPGet: &corev1.HTTPGetAction{ Path: coh.DefaultLivenessPath, - Port: intstr.FromInt(int(coh.DefaultHealthPort)), + Port: intstr.FromInt32(coh.DefaultHealthPort), Scheme: "HTTP", }, TCPSocket: nil, @@ -261,7 +261,7 @@ func TestCreateStatefulSetWithLivenessProbeSpecWithHttpGet(t *testing.T) { handler := &corev1.HTTPGetAction{ Path: "/test/ready", - Port: intstr.FromInt(1234), + Port: intstr.FromInt32(1234), } probe := coh.ReadinessProbeSpec{ @@ -303,7 +303,7 @@ func TestCreateStatefulSetWithLivenessProbeSpecWithHttpGet(t *testing.T) { func TestCreateStatefulSetWithLivenessProbeSpecWithTCPSocket(t *testing.T) { handler := &corev1.TCPSocketAction{ - Port: intstr.FromInt(1234), + Port: intstr.FromInt32(1234), Host: "foo.com", } @@ -442,7 +442,7 @@ func TestCreateStatefulSetWithStartupProbeSpec(t *testing.T) { Exec: nil, HTTPGet: &corev1.HTTPGetAction{ Path: coh.DefaultLivenessPath, - Port: intstr.FromInt(int(coh.DefaultHealthPort)), + Port: intstr.FromInt32(coh.DefaultHealthPort), Scheme: "HTTP", }, TCPSocket: nil, @@ -462,7 +462,7 @@ func TestCreateStatefulSetWithStartupProbeSpecWithHttpGet(t *testing.T) { handler := &corev1.HTTPGetAction{ Path: "/test/ready", - Port: intstr.FromInt(1234), + Port: intstr.FromInt32(1234), } probe := coh.ReadinessProbeSpec{ @@ -504,7 +504,7 @@ func TestCreateStatefulSetWithStartupProbeSpecWithHttpGet(t *testing.T) { func TestCreateStatefulSetWithStartupProbeSpecWithTCPSocket(t *testing.T) { handler := &corev1.TCPSocketAction{ - Port: intstr.FromInt(1234), + Port: intstr.FromInt32(1234), Host: "foo.com", } diff --git a/api/v1/create_statefulset_test.go b/api/v1/create_statefulset_test.go index 116b91aa2..9c1f2a659 100644 --- a/api/v1/create_statefulset_test.go +++ b/api/v1/create_statefulset_test.go @@ -170,8 +170,8 @@ func TestCreateStatefulSetWithHealthPort(t *testing.T) { deployment := createTestDeployment(spec) // Create expected StatefulSet stsExpected := createMinimalExpectedStatefulSet(deployment) - stsExpected.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Port = intstr.FromInt(210) - stsExpected.Spec.Template.Spec.Containers[0].LivenessProbe.HTTPGet.Port = intstr.FromInt(210) + stsExpected.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Port = intstr.FromInt32(210) + stsExpected.Spec.Template.Spec.Containers[0].LivenessProbe.HTTPGet.Port = intstr.FromInt32(210) // assert that the StatefulSet is as expected assertStatefulSetCreation(t, deployment, stsExpected) diff --git a/api/v1/create_wka_services_test.go b/api/v1/create_wka_services_test.go index 34cebcd03..7bf487960 100644 --- a/api/v1/create_wka_services_test.go +++ b/api/v1/create_wka_services_test.go @@ -50,15 +50,8 @@ func TestCreateWKAServiceForMinimalDeployment(t *testing.T) { ClusterIP: corev1.ClusterIPNone, // Pods must be part of the WKA service even if not ready PublishNotReadyAddresses: true, - Ports: []corev1.ServicePort{ - { - Name: "tcp-" + coh.PortNameCoherence, - Protocol: corev1.ProtocolTCP, - Port: 7, - TargetPort: intstr.FromInt(7), - }, - }, - Selector: selector, + Ports: getDefaultServicePorts(), + Selector: selector, }, } @@ -105,15 +98,8 @@ func TestCreateWKAServiceWithAppLabel(t *testing.T) { ClusterIP: corev1.ClusterIPNone, // Pods must be part of the WKA service even if not ready PublishNotReadyAddresses: true, - Ports: []corev1.ServicePort{ - { - Name: "tcp-" + coh.PortNameCoherence, - Protocol: corev1.ProtocolTCP, - Port: 7, - TargetPort: intstr.FromInt(7), - }, - }, - Selector: selector, + Ports: getDefaultServicePorts(), + Selector: selector, }, } @@ -160,15 +146,8 @@ func TestCreateWKAServiceWithVersionLabel(t *testing.T) { ClusterIP: corev1.ClusterIPNone, // Pods must be part of the WKA service even if not ready PublishNotReadyAddresses: true, - Ports: []corev1.ServicePort{ - { - Name: "tcp-" + coh.PortNameCoherence, - Protocol: corev1.ProtocolTCP, - Port: 7, - TargetPort: intstr.FromInt(7), - }, - }, - Selector: selector, + Ports: getDefaultServicePorts(), + Selector: selector, }, } @@ -212,15 +191,8 @@ func TestCreateWKAServiceForDeploymentWithClusterName(t *testing.T) { ClusterIP: corev1.ClusterIPNone, // Pods must be part of the WKA service even if not ready PublishNotReadyAddresses: true, - Ports: []corev1.ServicePort{ - { - Name: "tcp-" + coh.PortNameCoherence, - Protocol: corev1.ProtocolTCP, - Port: 7, - TargetPort: intstr.FromInt(7), - }, - }, - Selector: selector, + Ports: getDefaultServicePorts(), + Selector: selector, }, } @@ -241,3 +213,36 @@ func assertWKAService(t *testing.T, deployment *coh.Coherence, expected *corev1. diffs := deep.Equal(resActual, resExpected) g.Expect(diffs).To(BeNil()) } + +func getDefaultServicePorts() []corev1.ServicePort { + return []corev1.ServicePort{ + { + Name: coh.PortNameCoherence, + Protocol: corev1.ProtocolTCP, + AppProtocol: pointer.String(coh.AppProtocolTcp), + Port: 7, + TargetPort: intstr.FromInt32(7), + }, + { + Name: coh.PortNameCoherenceLocal, + Protocol: corev1.ProtocolTCP, + AppProtocol: pointer.String(coh.AppProtocolTcp), + Port: coh.DefaultUnicastPort, + TargetPort: intstr.FromString(coh.PortNameCoherenceLocal), + }, + { + Name: coh.PortNameCoherenceCluster, + Protocol: corev1.ProtocolTCP, + AppProtocol: pointer.String(coh.AppProtocolTcp), + Port: coh.DefaultClusterPort, + TargetPort: intstr.FromString(coh.PortNameCoherenceCluster), + }, + { + Name: coh.PortNameHealth, + Protocol: corev1.ProtocolTCP, + AppProtocol: pointer.String(coh.AppProtocolHttp), + Port: coh.DefaultHealthPort, + TargetPort: intstr.FromString(coh.PortNameHealth), + }, + } +} diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index e8d0d2c2f..db03cd762 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -1326,6 +1326,11 @@ func (in *NamedPortSpec) DeepCopyInto(out *NamedPortSpec) { *out = new(ServiceMonitorSpec) (*in).DeepCopyInto(*out) } + if in.ExposeOnSTS != nil { + in, out := &in.ExposeOnSTS, &out.ExposeOnSTS + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamedPortSpec. diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index e13c409ab..fe9129e2f 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -6,8 +6,10 @@ metadata: labels: control-plane: coherence app.kubernetes.io/name: coherence-operator + app: coherence-operator app.kubernetes.io/instance: coherence-operator-manager app.kubernetes.io/version: "3.3.2" + version: "3.3.2" app.kubernetes.io/component: manager app.kubernetes.io/part-of: coherence-operator spec: @@ -20,8 +22,10 @@ spec: labels: control-plane: coherence app.kubernetes.io/name: coherence-operator + app: coherence-operator app.kubernetes.io/instance: coherence-operator-manager app.kubernetes.io/version: "3.3.2" + version: "3.3.2" app.kubernetes.io/component: manager app.kubernetes.io/part-of: coherence-operator spec: diff --git a/controllers/coherence_controller.go b/controllers/coherence_controller.go index 428e1efd2..99be2fc2c 100644 --- a/controllers/coherence_controller.go +++ b/controllers/coherence_controller.go @@ -10,6 +10,7 @@ import ( "context" "fmt" "github.com/go-logr/logr" + coh "github.com/oracle/coherence-operator/api/v1" "github.com/oracle/coherence-operator/controllers/predicates" "github.com/oracle/coherence-operator/controllers/reconciler" "github.com/oracle/coherence-operator/controllers/secret" @@ -35,8 +36,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" "strconv" - - coh "github.com/oracle/coherence-operator/api/v1" ) const ( @@ -221,7 +220,7 @@ func (in *CoherenceReconciler) Reconcile(ctx context.Context, request ctrl.Reque // The "storeHash" is not "", so it must have been processed by the Operator (could have been a previous version). // There was a bug prior to 3.2.8 where the hash was calculated at the wrong point in the defaulting web-hook, // so the "currentHash" may be wrong, and hence differ from the recalculated "hash". - if deployment.IsBeforeVersion("3.2.8") { + if deployment.IsBeforeVersion("3.3.0") { // the AnnotationOperatorVersion annotation was added in the 3.2.8 web-hook, so if it is missing // the Coherence resource was added or updated prior to 3.2.8 // In this case we just ignore the difference in hash. @@ -282,6 +281,9 @@ func (in *CoherenceReconciler) Reconcile(ctx context.Context, request ctrl.Reque } return reconcile.Result{}, fmt.Errorf("one or more secondary resource reconcilers failed to reconcile") } + //} else { + // log.Info("Skipping updates for Coherence resource, annotation " + coh.AnnotationOperatorIgnore + " is set to true") + //} // if replica count is zero update the status to Stopped if deployment.GetReplicas() == 0 { @@ -348,11 +350,11 @@ func (in *CoherenceReconciler) ensureHashApplied(ctx context.Context, c *coh.Coh hash, _ := coh.EnsureHashLabel(latest) if currentHash != hash { - if c.IsBeforeVersion("3.2.8") { - // Before 3.2.8 there was a bug calculating the has in the defaulting web-hook + if c.IsBeforeVersion("3.3.0") { + // Before 3.3.0 there was a bug calculating the has in the defaulting web-hook // This would cause the hashes to be different here, when in fact they should not be // If the Coherence resource being processes has no version annotation, or a version - // prior to 3.2.8 then we return as if the hashes matched + // prior to 3.3.0 then we return as if the hashes matched if labels == nil { labels = make(map[string]string) } diff --git a/controllers/reconciler/base_controller.go b/controllers/reconciler/base_controller.go index 46759a151..1d9663de2 100644 --- a/controllers/reconciler/base_controller.go +++ b/controllers/reconciler/base_controller.go @@ -812,9 +812,9 @@ func (in *ReconcileSecondaryResource) ReconcileSingleResource(ctx context.Contex } switch { - case owner == nil || owner.GetReplicas() == 0: + case owner == nil: if exists { - // The owning Coherence resource does not exist (or is scaled down to zero) but the resource still does, + // The owning Coherence resource does not exist but the resource still does, // ensure that the resource is deleted. // This should not actually be required as everything is owned by the owning Coherence resource // and there should be a cascaded delete by k8s, so it's belt and braces. diff --git a/docs/about/04_coherence_spec.adoc b/docs/about/04_coherence_spec.adoc index 044cb2d99..28c18b7d7 100644 --- a/docs/about/04_coherence_spec.adoc +++ b/docs/about/04_coherence_spec.adoc @@ -474,6 +474,7 @@ m| hostPort | Number of port to expose on the host. If specified, this must be a m| hostIP | What host IP to bind the external port to. m| *string | false m| service | Service configures the Kubernetes Service used to expose the port. m| *<> | false m| serviceMonitor | The specification of a Prometheus ServiceMonitor resource that will be created for the Service being exposed for this port. m| *<> | false +m| exposeOnSts | ExposeOnSTS is a flag to indicate that this port should also be exposed on the StatefulSetHeadless service. This is useful in cases where a service mesh such as Istio is being used and ports such as the Extend or gRPC ports are accessed via the StatefulSet service. The default is `true` so all additional ports are exposed on the StatefulSet headless service. m| *bool | false |=== <> diff --git a/examples/015_simple_image/README.adoc b/examples/015_simple_image/README.adoc index 7cef11e3a..52eaa1c4c 100644 --- a/examples/015_simple_image/README.adoc +++ b/examples/015_simple_image/README.adoc @@ -52,7 +52,7 @@ In the `build.gradle` file we add the bom as a platform dependency. .build.gradle ---- dependencies { - implementation platform("com.oracle.coherence.ce:coherence-bom:22.06.4") + implementation platform("com.oracle.coherence.ce:coherence-bom:22.06.6") ---- We can then add the `coherence` and `coherence-json` modules as dependencies @@ -77,7 +77,7 @@ In the `build.gradle` file we add the bom as a platform dependency. .build.gradle ---- dependencies { - implementation platform("com.oracle.coherence.ce:coherence-bom:22.06.4") + implementation platform("com.oracle.coherence.ce:coherence-bom:22.06.6") implementation "com.oracle.coherence.ce:coherence" implementation "com.oracle.coherence.ce:coherence-json" diff --git a/examples/015_simple_image/build.gradle b/examples/015_simple_image/build.gradle index 34aa6ac9b..5beefb71e 100644 --- a/examples/015_simple_image/build.gradle +++ b/examples/015_simple_image/build.gradle @@ -19,7 +19,7 @@ repositories { } dependencies { - implementation platform("com.oracle.coherence.ce:coherence-bom:22.06.4") + implementation platform("com.oracle.coherence.ce:coherence-bom:22.06.6") implementation "com.oracle.coherence.ce:coherence" implementation "com.oracle.coherence.ce:coherence-json" diff --git a/examples/015_simple_image/pom.xml b/examples/015_simple_image/pom.xml index f0ec0c19c..ffed3b645 100644 --- a/examples/015_simple_image/pom.xml +++ b/examples/015_simple_image/pom.xml @@ -24,7 +24,7 @@ 1111 - 22.06.4 + 22.06.63.3.2 diff --git a/examples/016_simple_docker_image/README.adoc b/examples/016_simple_docker_image/README.adoc index ded54a069..714fd9b0e 100644 --- a/examples/016_simple_docker_image/README.adoc +++ b/examples/016_simple_docker_image/README.adoc @@ -205,7 +205,7 @@ The `image-assembly.xml` descriptor file is shown below, and configures the foll * There are two `` configured: ** The first copies any class files in `target/classes` to `app/classes` (which will actually be `target/docker/app/classes`) ** The second copies all files under `src/docker` (i.e. the `Dockerfile`) into `target/docker` -* The `` configuration copies all the project dependencies (including transitive dependencies) to the `app/libs` directory (actually the `target/docker/app/libs` directory). Any version information will be stripped from the files, so `coherence-22.06.4.jar` would become `coherence.jar`. +* The `` configuration copies all the project dependencies (including transitive dependencies) to the `app/libs` directory (actually the `target/docker/app/libs` directory). Any version information will be stripped from the files, so `coherence-22.06.6.jar` would become `coherence.jar`. [source,xml] .src/assembly/image-assembly.xml @@ -274,7 +274,7 @@ In the `build.gradle` file we add the bom as a platform dependency and then add .build.gradle ---- dependencies { - implementation platform("com.oracle.coherence.ce:coherence-bom:22.06.4") + implementation platform("com.oracle.coherence.ce:coherence-bom:22.06.6") implementation "com.oracle.coherence.ce:coherence" implementation "com.oracle.coherence.ce:coherence-json" diff --git a/examples/016_simple_docker_image/build.gradle b/examples/016_simple_docker_image/build.gradle index d5a8cd65a..72680aef4 100644 --- a/examples/016_simple_docker_image/build.gradle +++ b/examples/016_simple_docker_image/build.gradle @@ -18,7 +18,7 @@ repositories { } dependencies { - implementation platform("com.oracle.coherence.ce:coherence-bom:22.06.4") + implementation platform("com.oracle.coherence.ce:coherence-bom:22.06.6") implementation "com.oracle.coherence.ce:coherence" implementation "com.oracle.coherence.ce:coherence-json" diff --git a/examples/016_simple_docker_image/pom.xml b/examples/016_simple_docker_image/pom.xml index 01a191652..ad1ebf480 100644 --- a/examples/016_simple_docker_image/pom.xml +++ b/examples/016_simple_docker_image/pom.xml @@ -24,7 +24,7 @@ 11 11 - 22.06.4 + 22.06.6 3.3.2 diff --git a/examples/021_deployment/pom.xml b/examples/021_deployment/pom.xml index 68fe44a00..e83107a9d 100644 --- a/examples/021_deployment/pom.xml +++ b/examples/021_deployment/pom.xml @@ -16,7 +16,7 @@ com.oracle.coherence.ce - 22.06.4 + 22.06.6 1.3.1 diff --git a/examples/025_extend_client/build.gradle b/examples/025_extend_client/build.gradle index c2adeec64..a301ab78c 100644 --- a/examples/025_extend_client/build.gradle +++ b/examples/025_extend_client/build.gradle @@ -19,7 +19,7 @@ repositories { } dependencies { - implementation platform("com.oracle.coherence.ce:coherence-bom:22.06.4") + implementation platform("com.oracle.coherence.ce:coherence-bom:22.06.6") implementation "com.oracle.coherence.ce:coherence" } diff --git a/examples/025_extend_client/pom.xml b/examples/025_extend_client/pom.xml index ecfab1a0b..b7dd22b5d 100644 --- a/examples/025_extend_client/pom.xml +++ b/examples/025_extend_client/pom.xml @@ -24,7 +24,7 @@ 11 11 - 22.06.4 + 22.06.6 3.3.2 3.0.0 diff --git a/examples/090_tls/pom.xml b/examples/090_tls/pom.xml index 15da76570..67ccfee4f 100644 --- a/examples/090_tls/pom.xml +++ b/examples/090_tls/pom.xml @@ -26,7 +26,7 @@ 11 com.oracle.coherence.ce - 22.06.4 + 22.06.6 3.3.2 diff --git a/examples/200_autoscaler/pom.xml b/examples/200_autoscaler/pom.xml index fc9910b67..cf50b9f77 100644 --- a/examples/200_autoscaler/pom.xml +++ b/examples/200_autoscaler/pom.xml @@ -20,7 +20,7 @@ 11 com.oracle.coherence.ce - 22.06.4 + 22.06.6 3.3.2 diff --git a/examples/400_Istio/README.adoc b/examples/400_Istio/README.adoc index a46268a23..a4859b3b2 100644 --- a/examples/400_Istio/README.adoc +++ b/examples/400_Istio/README.adoc @@ -1,6 +1,6 @@ /////////////////////////////////////////////////////////////////////////////// - Copyright (c) 2021, Oracle and/or its affiliates. + Copyright (c) 2021, 2023, Oracle and/or its affiliates. Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. @@ -9,19 +9,51 @@ == Using Coherence with Istio -You can run the Coherence cluster and manage then using the Coherence Operator alongside https://istio.io[Istio]. -Coherence clusters managed with the Coherence Operator 3.2.0 and later work with Istio 1.9.1 and later. -Coherence caches can be accessed from outside the Coherence cluster via Coherence*Extend, REST, and other supported Coherence clients. +You can run the Coherence cluster and manage them using the Coherence Operator alongside https://istio.io[Istio]. +Coherence clusters managed with the Coherence Operator 3.3.2 and later work with Istio 1.9.1 and later out of the box. +Coherence caches can be accessed from outside the Coherence cluster via Coherence*Extend, REST, and other supported +Coherence clients. Using Coherence clusters with Istio does not require the Coherence Operator to also be using Istio (and vice-versa) . The Coherence Operator can manage Coherence clusters independent of whether those clusters are using Istio or not. -=== Why Doesn't Coherence Work with Istio? - -Coherence uses a custom TCP message protocol for inter-cluster member communication. -When a cluster member sends a message to another member, the "reply to" address of the sending member is in the message. This address is the socket address the member is listening on (i.e. it is the IP address and port Coherence has bound to). -When Istio is intercepting traffic the message ends up being sent via the Envoy proxy and the actual port Coherence is listening on is blocked by Istio. When the member that receives the message tries to send a response to the reply to address, that port is not visible to it. - -Coherence clients will work with Istio, so Extend, gRPC and http clients for things like REST, metrics and management will work when routed through the Envoy proxy. +Although Coherence itself can be configured to use TLS, when using Istio Coherence cluster members and clients can +just use the default socket configurations and Istio will control and route all the traffic over mTLS. + +[TIP] +==== +Coherence clusters can be manually configured to work with Istio, even if not using the Operator. +See the Istio example in the <> +==== + +=== How Does Coherence Work with Istio? + +Istio is a "Service Mesh" so the clue to how Istio works in Kubernetes is in the name, it relies on the configuration +of Kubernetes Services. +This means that any ports than need to be accessed in Pods, including those using in "Pod to Pod" communication +must be exposed via a Service. Usually a Pod can reach any port on another Pod even if it is not exposed in the +container spec, but this is not the case when using Istio as only ports exposed by the Envoy proxy are allowed. + +For Coherence cluster membership, this means the cluster port and the local port must be exposed on a Service. +To do this the local port must be configured to be a fixed port instead of the default ephemeral port. +The Coherence Operator uses the default cluster port of `7574` and there is no reason to ever change this. +The Coherence Operator always configures a fixed port for the local port so this works with Istio out of the box. +In addition, the Operator uses the health check port to determine the status of a cluster, so this needs to be +exposed so that the Operator can reach Coherence Pods. + +The Coherence localhost property can be set to the name of the Pod. +This is easily done using the container environment variables, which the Operator does automatically. + +Coherence clusters are run as a StatefulSet in Kubernetes. This means that the Pods are configured with a host name +and a subdomain based on the name of the StatefulSet headless service name, and it is this name that should be used +to access Pods. +For example for a Coherence resource named `storage` the Operator will create a StatefulSet named `storgage` with a +headless service named `storage-sts`. Each Pod in a StatefulSet is numbered with a fixed identity, so the first Pod +in this cluster will be `storage-0`. The Pod has a number of DNS names that it is reachable with, but the fully +qualified name using the headless service will be `storage-0.storage-sts` or storage-0.storage-sts..svc`. + +By default, the Operator will expose all the ports configured for the `Coherence` resource on the StatefulSet headless +service. This allows Coherence Extend and gRPC clients to use this service name as the WKA address when using the +Coherence NameService to lookup endpoints (see the client example below). === Prerequisites @@ -29,7 +61,10 @@ The instructions assume that you are using a Kubernetes cluster with Istio insta ==== Enable Istio Strict Mode -For this example we make Istio run in "strict" mode so that it will not allow any traffic between Pods outside the Envoy proxy. If other modes are used, such as permissive, then Coherence will work as normal as its ports will not be blocked. +For this example we make Istio run in "strict" mode so that it will not allow any traffic between Pods outside the +Envoy proxy. +If other modes are used, such as permissive, then Istio allows Pod to Pod communication so a cluster may appear to work +in permissive mode, when it would not in strict mode. To set Istio to strict mode create the following yaml file. @@ -54,7 +89,10 @@ kubectl -n istio-system apply istio-strict.yaml === Using the Coherence operator with Istio -To use Coherence operator with Istio, you can deploy the operator into a namespace which has Istio automatic sidecar injection enabled. Before installing the operator, create the namespace in which you want to run the Coherence operator and label it for automatic injection. +To use Coherence operator with Istio, you can deploy the operator into a namespace which has Istio automatic sidecar +injection enabled. +Before installing the operator, create the namespace in which you want to run the Coherence operator and label it for +automatic injection. [source,bash] ---- @@ -71,7 +109,10 @@ This web-hook binds to port `9443` in the Operator Pods and is already configure Kubernetes admissions web-hooks. If this port is routed through the Envoy proxy Kubernetes will be unable to access the web-hook. -There are a number of ways to exclude the web-hook port, the simplest is to add a `PeerAuthentication` resource to the Operator namespace. +The Operator yaml manifests and Helm chart already add the `traffic.sidecar.istio.io/excludeInboundPorts` annotation +to the Operator Pods. This should exclude the web-hook port from being Istio. + +Another way to do this is to add a `PeerAuthentication` resource to the Operator namespace. *Before installing the Operator*, create the following `PeerAuthentication` yaml. @@ -103,7 +144,8 @@ For example, if the Operator will be in the `coherence` namespace: kubectl -n coherence apply istio-operator.yaml ---- -You can then install the operator using your preferred method in the Operator <>. +You can then install the operator using your preferred method in the +Operator <>. After installed operator, use the following command to confirm the operator is running: @@ -115,11 +157,21 @@ NAME READY STATUS RESTA coherence-operator-controller-manager-7d76f9f475-q2vwv 2/2 Running 1 17h ---- -The output should show 2/2 in READY column, meaning there are 2 containers running in the Operator pod. One is Coherence Operator and the other is Envoy Proxy. +The output should show 2/2 in READY column, meaning there are 2 containers running in the Operator pod. +One is Coherence Operator and the other is Envoy Proxy. + +If we use the Istio Kiali addon to visualize Istio we can see the Operator in the list of applications + +image::images/kiali-operator-app.png[width=1024,height=512] + +We can also see on the detailed view, that the Operator talks to the Kubernetes API server + +image::images/kiali-operator-app-graph.png[width=1024,height=512] === Creating a Coherence cluster with Istio -You can configure your cluster to run with Istio automatic sidecar injection enabled. Before creating your cluster, create the namespace in which you want to run the cluster and label it for automatic injection. +You can configure a cluster to run with Istio automatic sidecar injection enabled. Before creating the cluster, +create the namespace in which the cluster will run and label it for automatic injection. [source,bash] ---- @@ -127,198 +179,210 @@ kubectl create namespace coherence-example kubectl label namespace coherence-example istio-injection=enabled ---- -==== Exclude the Coherence Cluster Ports - -As explained above, Coherence cluster traffic must be excluded from the Envoy proxy, there are various ways to do this. - -There are three ports that must be excluded: +Now create a Coherence resource as normal, there is no additional configuration required to work in Istio. -* The cluster port - defaults to 7574, there is no need to set this to any other value. -* The TCP first local port - the Operator will default this to 7575 using its web-hook (if the web-hook is disabled this needs to be manually set). -* The TCP second local port - the Operator will default this to 7576 using its web-hook (if the web-hook is disabled this needs to be manually set). - -*1 Use an Annotation in the Coherence Resource* - -The Istio exclusion annotation `traffic.sidecar.istio.io/excludeInboundPorts` can be added to the Coherence yaml to list the ports to be excluded, - -For example, using the default ports the following annotation will exclude those ports from Istio: +For example using the yaml below to create a three member cluster with management and metrics enabled: [source,yaml] -.coherence-storage.yaml +.storage.yaml ---- apiVersion: coherence.oracle.com/v1 kind: Coherence metadata: name: storage spec: - annotations: - traffic.sidecar.istio.io/excludeInboundPorts: "7574,7575,7576" + replicas: 3 + image: ghcr.io/oracle/coherence-ce:22.06.6 + labels: + app: storage # <1> + version: 1.0.0 # <2> + coherence: + management: + enabled: true + metrics: + enabled: true + ports: + - name: management # <3> + - name: metrics + - name: extend + port: 20000 + appProtocol: tcp # <4> + - name: grpc-proxy + port: 1408 + appProtocol: grpc # <5> ---- -If the Coherence Operator's web-hook has been disabled, the local ports must be set in the yaml too: +<1> Istio prefers applications to have an `app` label +<2> Istio prefers applications to have a `version` label +<3> The Coherence Pods will expose ports for Management over REST, metrics, a Coherence*Extend proxy and a gRPC proxy +<4> The Operator will set the `appProtocol` for the management and metrics ports to `http`, but the Extend port must be +set manually to `tcp` so that Istio knows what sort of traffic is being used by that port +<5> The gRPC port's `appProtocol` field is set to `grpc` -[source,yaml] -.coherence-storage.yaml +Using the Kiali console, we can now see two applications, the Coherence Operator in the "coherence" namespace +and the "storage" application in the "coherence-example" namespace. + +image::images/kiali-storage-app.png[width=1024,height=512] + +If we look at the graph view we can see all the traffic between the different parts of the system + +image::images/kiali-post-deploy.png[width=1024,height=512] + +- We can see the Kubernetes API server accessing the Operator web-hook to validate the yaml +- We can see tge storage pods (the box marked "storage 1.0.0") communicate with each other via the storage-sts service to from a Coherence cluster +- We can see the storage pods communicate with the Operator REST service to request their Coherence site and rack labels +- We can see the Operator ping the storage pods health endpoints via the storage-sts service + +All of this traffic is using mTLS controlled by Istio + +=== Coherence Clients Running in Kubernetes + +Coherence Extend clients and gRPC clients running inside the cluster will also work with Istio. + +For this example the clients will be run in the `coherence-client` namespace, so it needs to be +created and labelled so that Istio injection works in that namespace. + +[source,bash] ---- -apiVersion: coherence.oracle.com/v1 -kind: Coherence -metadata: - name: storage -spec: - annotations: - traffic.sidecar.istio.io/excludeInboundPorts: "7574,7575,7576" - coherence: - localPort: 7575 - localPortAdjust: 7576 +kubectl create namespace coherence-client +kubectl label namespace coherence-client istio-injection=enabled ---- -*2 Use a PeerAuthentication resource* +To simulate a client application a `CoherenceJob` resource will be used with different configurations +for the different types of client. + +The simplest way to configure a Coherence extend client in a cache configuration file is a default configuration +similar to that shown below. No ports or addresses need to be configured. Coherence will use the JVM's configured +cluster name and well know addresses to locate to look up the Extend endpoints using the Coherence NameService. + +[source,xml] +---- + + thin-remote + RemoteCache + Proxy + +---- -A `PeerAuthentication` resource can be added to the Coherence cluster's namespace *before the cluster is deployed*. +We can configure a `CoherenceJob` to run an Extend client with this configuration as shown below: [source,yaml] -.istio-coherence.yaml +.extend-client.yaml ---- -apiVersion: security.istio.io/v1beta1 -kind: PeerAuthentication +apiVersion: coherence.oracle.com/v1 +kind: CoherenceJob metadata: - name: "coherence" + name: client spec: - selector: - matchLabels: - coherenceComponent: coherencePod - mtls: - mode: STRICT - portLevelMtls: - 7574: - mode: PERMISSIVE - 7575: - mode: PERMISSIVE - 7576: - mode: PERMISSIVE + image: ghcr.io/oracle/coherence-ce:22.06.6 # <1> + restartPolicy: Never + cluster: storage # <2> + coherence: + wka: + addresses: + - "storage-sts.coherence-example.svc" # <3> + application: + type: operator # <4> + args: + - sleep + - "300s" + env: + - name: COHERENCE_CLIENT # <5> + value: "remote" + - name: COHERENCE_PROFILE # <6> + value: "thin" +---- + +<1> The client will use the CE image published on GitHub, which will use the default cache configuration file from Coherence jar. +<2> The cluster name must be set to the cluster name of the cluster started above, in this case `storage` +<3> The WKA address needs to be set to the DNS name of the headless service for the storage cluster created above. As this +Job is running in a different name space this is the fully qualified name `..svc` which is `storage-sts.coherence-example.svc` +<4> Instead of running a normal command this Job will run the Operator's `sleep` command and sleep for `300s` (300 seconds). +<5> The `COHERENCE_CLIENT` environment variable value of `remote` sets the Coherence cache configuration to be an Extend client using the NameService +<6> The `COHERENCE_PROFILE` environment variable value of `thin` sets the Coherence cache configuration not to use a Near Cache. + +The yaml above can be deployed into Kubernetes: + +[source,bash] +---- +kubectl -n coherence-client apply -f extend-client.yaml ---- -The Coherence Operator labels Coherence Pods with the label `coherenceComponent: coherencePod` so this can be used in the `PeerAuthentication`. Then each port to be excluded is listed in the `portLevelMtls` and set to be `PERMISSIVE`. +[source,bash] +---- +$ kubectl -n coherence-client get pod +NAME READY STATUS RESTARTS AGE +client-qgnw5 2/2 Running 0 80s +---- -This yaml can then be installed into the namespace that the Coherence cluster will be deployed into. +The Pod is now running but not doing anything, just sleeping. +If we look at the Kiali dashboard we can see the client application started and communicated wth the Operator. -=== TLS +image::images/kiali-client-started-graph.png[width=1024,height=512] -Coherence clusters work with mTLS and Coherence clients can also support TLS through the Istio Gateway with TLS termination to connect to Coherence cluster running inside kubernetes. For example, you can apply the following Istio Gateway and Virtual Service in the namespace of the Coherence cluster. Before applying the gateway, create a secret for the credential from the certificate and key (e.g. server.crt and server.key) to be used by the Gateway: +We can use this sleeping Pod to exec into and run commands. In this case we will create a Coherence QueryPlus +client and run some CohQL commands. The command below will exec into the sleeping Pod. [source,bash] ---- -kubectl create -n istio-system secret tls extend-credential --key=server.key --cert=server.crt +kubectl -n coherence-client exec -it client-qgnw5 -- /coherence-operator/utils/runner queryplus ---- -Then, create a keystore (server.jks) to be used by the Coherence Extend client, e.g.: +A QueryPlus client will be started and eventually display the `CohQL>` prompt. + [source,bash] ---- -openssl pkcs12 -export -in server.crt -inkey server.key -chain -CAfile ca.crt -name "server" -out server.p12 +Coherence Command Line Tool -keytool -importkeystore -deststorepass password -destkeystore server.jks -srckeystore server.p12 -srcstoretype PKCS12 +CohQL> ---- +A simple command to try is just creating a cache, so at the prompt type the command `create cache test` which will +create a cache named `test`. If all is configured correctly this client will connect to the cluster over Extend +and create the cache called `test` and return to the `CohQL` prompt. -tlsGateway.yaml -[source,bash] ----- -apiVersion: networking.istio.io/v1alpha3 -kind: Gateway -metadata: - name: tlsgateway -spec: - selector: - istio: ingressgateway # use istio default ingress gateway - servers: - - port: - number: 8043 - name: tls - protocol: TLS - tls: - mode: SIMPLE - credentialName: "extend-credential" # the secret created in the previous step - maxProtocolVersion: TLSV1_3 - hosts: - - "*" ----- - -tlsVS.yaml [source,bash] ---- -apiVersion: networking.istio.io/v1alpha3 -kind: VirtualService -metadata: - name: extend -spec: - hosts: - - "*" - gateways: - - tlsgateway - tcp: - - match: - route: - - destination: - host: example-cluster-proxy-proxy # the service name used to expose the Extend proxy port +Coherence Command Line Tool + +CohQL> create cache test ---- -Apply the Gateway and VirtualService: +We can also try selecting data from the cache using the CohQL query `select * from test` +(which will return nothing as the cache is empty). [source,bash] ---- -kubectl apply -f tlsGateway.yaml -n coherence-example -kubectl apply -f tlsVS.yaml -n coherence-example ----- - -Then configure a Coherence*Extend client to connect to the proxy server via TLS protocol. Below is an example of a configuration of an Extend client using TLS port 8043 configured in the Gateway and server.jks created earlier in the example. - -client-cache-config.xml ----- -... - - extend-direct - ExtendTcpProxyService - - - - - TLS - - PeerX509 - - file:server.jks - password - - - - - - -
$INGRESS_HOST
- 8043 - - - - - -... ----- - -If you are using Docker for Desktop, `$INGRESS_HOST` is `127.0.0.1`, and you can use the Kubectl port-forward to allow the Extend client to access the Coherence cluster from your localhost: +CohQL> select * from test +Results -[source,bash] ----- -kubectl port-forward -n istio-system 8043:8043 +CohQL> ---- -=== Prometheus +If we now look at the Kiali dashboard we can see that the client has communicated with the storage cluster. +All of this communication was using mTLS but without configuring Coherence to use TLS. + +image::images/kiali-client-storage.png[width=1024,height=512] + +To exit from the `CohQL>` prompt type the `bye` command. + + +Coherence Extend clients can connect to the cluster also using Istio to provide mTLS support. +Coherence clusters work with mTLS and Coherence clients can also support TLS through the Istio Gateway with TLS +termination to connect to Coherence cluster running inside kubernetes. +For example, you can apply the following Istio Gateway and Virtual Service in the namespace of the Coherence cluster. +Before applying the gateway, create a secret for the credential from the certificate and key +(e.g. server.crt and server.key) to be used by the Gateway: -The coherence metrics that record and track the health of Coherence cluster using Prometheus are also available in Istio environment and can be viewed through Grafana. However, Coherence cluster traffic is not visible by Istio. -=== Traffic Visualization -Istio provides traffic management capabilities, including the ability to visualize traffic in https://kiali.io[Kiali]. You do not need to change your applications to use this feature. The Istio proxy (envoy) sidecar that is injected into your pods provides it. The image below shows an example with traffic flow. In this example, you can see how the traffic flows in from the Istio gateway on the left, to the cluster services, and then to the individual cluster members. This example has storage members (example-cluster-storage), a proxy member running proxy service (example-cluster-proxy), and a REST member running http server (example-cluster-rest). However, Coherence cluster traffic between members is not visible. +=== Coherence Clients Running Outside Kubernetes -image::images/istioKiali.png[width=1024,height=512] +Coherence clients running outside the Kubernetes can be configured to connect to a Coherence cluster inside +Kubernetes using any of the ingress or gateway features of Istio and Kubernetes. +All the different ways to do this are beyond the scope of this simple example as there are many and they +depend on the versions of Istio and Kubernetes being used. -To learn more, see https://istio.io/latest/docs/concepts/traffic-management/[Istio traffic management]. +When connecting Coherence Extend or gRPC clients from outside Kubernetes, the Coherence NameService cannot be used +by clients to look up the endpoints. The clients must be configured with fixed endpoints using the hostnames and ports +of the configured ingress or gateway services. diff --git a/examples/400_Istio/images/kiali-client-started-graph.png b/examples/400_Istio/images/kiali-client-started-graph.png new file mode 100644 index 000000000..17533c9df Binary files /dev/null and b/examples/400_Istio/images/kiali-client-started-graph.png differ diff --git a/examples/400_Istio/images/kiali-client-storage.png b/examples/400_Istio/images/kiali-client-storage.png new file mode 100644 index 000000000..7521caaab Binary files /dev/null and b/examples/400_Istio/images/kiali-client-storage.png differ diff --git a/examples/400_Istio/images/kiali-operator-app-graph.png b/examples/400_Istio/images/kiali-operator-app-graph.png new file mode 100644 index 000000000..f2d956601 Binary files /dev/null and b/examples/400_Istio/images/kiali-operator-app-graph.png differ diff --git a/examples/400_Istio/images/kiali-operator-app.png b/examples/400_Istio/images/kiali-operator-app.png new file mode 100644 index 000000000..63e4e4846 Binary files /dev/null and b/examples/400_Istio/images/kiali-operator-app.png differ diff --git a/examples/400_Istio/images/kiali-post-deploy.png b/examples/400_Istio/images/kiali-post-deploy.png new file mode 100644 index 000000000..708860c24 Binary files /dev/null and b/examples/400_Istio/images/kiali-post-deploy.png differ diff --git a/examples/400_Istio/images/kiali-storage-app.png b/examples/400_Istio/images/kiali-storage-app.png new file mode 100644 index 000000000..d9d631b1e Binary files /dev/null and b/examples/400_Istio/images/kiali-storage-app.png differ diff --git a/examples/no-operator/04_istio/README.adoc b/examples/no-operator/04_istio/README.adoc new file mode 100644 index 000000000..c66907def --- /dev/null +++ b/examples/no-operator/04_istio/README.adoc @@ -0,0 +1,366 @@ +/////////////////////////////////////////////////////////////////////////////// + + Copyright (c) 2023, Oracle and/or its affiliates. + Licensed under the Universal Permissive License v 1.0 as shown at + http://oss.oracle.com/licenses/upl. + +/////////////////////////////////////////////////////////////////////////////// += Running Coherence with Istio + +== Running Coherence with Istio + +This example shows how to deploy a simple Coherence cluster in Kubernetes with Istio. + +Coherence can be configured to work with https://istio.io[Istio], even if Istio is configured in Strict Mode. +Coherence caches can be accessed from inside or outside the Kubernetes cluster via Coherence*Extend, REST, +and other supported Coherence clients. +Although Coherence itself can be configured to use TLS, when using Istio Coherence cluster members and clients can +just use the default socket configurations and Istio will control and route all the traffic over mTLS. + +== How Does Coherence Work with Istio? + +Istio is a "Service Mesh" so the clue to how Istio works in Kubernetes is in the name, it relies on the configuration +of Kubernetes Services. +This means that any ports than need to be accessed in Pods, including those using in "Pod to Pod" communication +must be exposed via a Service. Usually a Pod can reach any port on another Pod even if it is not exposed in the +container spec, but this is not the case when using Istio as only ports exposed by the Envoy proxy are allowed. + +For Coherence cluster membership, this means the cluster port and the local port must be exposed on a Service. +To do this the local port must be configured to be a fixed port instead of the default ephemeral port. +The default cluster port is `7574` and there is no reason to ever change this when running in containers. +A fixed local port has to be configured for Coherence to work with Istio out of the box. +Additional ports, management port, metrics port, etc. also need to be exposed if they are being used. + +Ideally, Coherence clusters are run as a StatefulSet in Kubernetes. +This means that the Pods are configured with a host name and a subdomain based on the name of the StatefulSet +headless service name, and it is this name that should be used to access Pods. + +=== Prerequisites + +The instructions assume that you are using a Kubernetes cluster with Istio installed and configured already. + +==== Enable Istio Strict Mode + +For this example we make Istio run in "strict" mode so that it will not allow any traffic between Pods outside the +Envoy proxy. +If other modes are used, such as permissive, then Istio allows Pod to Pod communication so a cluster may appear to work +in permissive mode, when it would not in strict mode. + +To set Istio to strict mode create the following yaml file. + +[source,yaml] +.istio-strict.yaml +---- +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: "default" +spec: + mtls: + mode: STRICT +---- + +Install this yaml into the Istio system namespace with the following command: + +[source,bash] +---- +kubectl -n istio-system apply istio-strict.yaml +---- + +== Create a Coherence Cluster + +The best way to run Coherence cluster members is to use a StatefulSet. Multiple StatefulSets can be created that +are all part of the same Coherence cluster. + +In this example we will run a Coherence cluster using the CE image. This image starts Coherence with health +checks enabled on port 6676, +an Extend proxy listening on port 20000, a gRPC proxy on port 1408, the cluster port set to 7574. +We will also enable Coherence Management over REST on port 30000, and metrics on port 9612. +We will set the Coherence local port to a fixed value of 7575. + +[NOTE] +==== +Istio has a few requirements for how Kubernetes resources are configured. +One of those is labels, where an `app` and `version` label are required to specify the application name +that the resource is part of and the version of that application. +All the resources in this example contains those labels. +==== + +=== Cluster Discovery Service + +For Coherence cluster discovery to work in Kubernetes we have to configure Coherence well-known-addresses which +requires a headless service. We cannot use the same headless service the we will create for the StatefulSet because +the WKA service must have the `publishNotReadyAddresses` field set to `true`, wheres the StatefulSet service does not. +We would not want the ports accessed via the StatefulSet service to route to unready Pods, but for cluster discovery +we must allow unready Pods to be part of the Service. + +The discovery service can be created with yaml like that shown below. + +[source,yaml] +.wka-service.yaml +---- +apiVersion: v1 +kind: Service +metadata: + name: storage-wka # <1> +spec: + clusterIP: None + publishNotReadyAddresses: true # <2> + selector: # <3> + app: my-coherence-app + version: 1.0.0 + ports: + - name: coherence # <4> + port: 7574 + targetPort: coherence + appProtocol: tcp +---- + +<1> The service name is `storeage-wka` and this will be used to configure the Coherence WKA address in the cluster. +<2> The `publishNotReadyAddresses` field must be set to `true` +<3> The `selector` is configured to match a sub-set of the Pod labels configured in the StatefulSet +<4> We do not really need or care about the port for the cluster discovery service, but all Kubernetes services must have +at least one port, so here we use the cluster port. We could use any random port, even one that nothing is listening on + +=== StatefulSet Headless Service + +All StatefulSets require a headless Service creating and the name of this Service is specified in the StatefulSet spec. +All the ports mentioned above will be exposed on this service. +The yaml for the service could look like this: + +[source,yaml] +.storage-service.yaml +---- +apiVersion: v1 +kind: Service +metadata: + name: storage-headless +spec: + clusterIP: None + selector: + app: my-coherence-app # <1> + version: 1.0.0 + ports: + - name: coherence # <2> + port: 7574 + targetPort: coherence + appProtocol: tcp + - name: coh-local # <3> + port: 7575 + targetPort: coh-local + appProtocol: tcp + - name: extend-proxy # <4> + port: 20000 + targetPort: extend-proxy + appProtocol: tcp + - name: grpc-proxy # <5> + port: 1408 + targetPort: grpc-proxy + appProtocol: grpc + - name: management # <6> + port: 30000 + targetPort: management + appProtocol: http + - name: metrics # <7> + port: 9612 + targetPort: metrics + appProtocol: http +---- + +<1> The selector labels will match a sub-set of the labels specified for the Pods in the StatefulSet +<2> The Coherence cluster port 7574 is exposed with the name `coherence` mapping to the container port in the StatefulSet named `coherence`. +This port has an `appProtocol` of `tcp` to tell Istio that the port traffic is raw TCP traffic. +<3> The Coherence local port 7575 is exposed with the name `coh-local` mapping to the container port in the StatefulSet named `coh-local` +This port has an `appProtocol` of `tcp` to tell Istio that the port traffic is raw TCP traffic. +<4> The Coherence Extend proxy port 20000 is exposed with the name `extend-proxy` mapping to the container port in the StatefulSet named `extend-proxy` +This port has an `appProtocol` of `tcp` to tell Istio that the port traffic is raw TCP traffic. +<5> The Coherence gRPC proxy port 1408 is exposed with the name `grpc-proxy` mapping to the container port in the StatefulSet named `grpc-proxy` +This port has an `appProtocol` of `grpc` to tell Istio that the port traffic is gRPC traffic. +<6> The Coherence Management over REST port 30000 is exposed with the name `management` mapping to the container port in the StatefulSet named `management` +This port has an `appProtocol` of `http` to tell Istio that the port traffic is http traffic. +<7> The Coherence Metrics port 9612 is exposed with the name `metrics` mapping to the container port in the StatefulSet named `metrics` +This port has an `appProtocol` of `http` to tell Istio that the port traffic is http traffic. + +[NOTE] +==== +Istio requires ports to specify the protocol used for their traffic, and this can be done in two ways. +Either using the `appProtocol` field for the ports, as shown above. +Or, prefix the port name with the protocol, so instead of `management` the port name would be `http-management` +==== + +=== The StatefulSet + +With the two Services defined, the StatefulSet can now be configured. +Istio + +[source,yaml] +.storage.yaml +---- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: storage + labels: + app: my-coherence-app + version: 1.0.0 +spec: + selector: + matchLabels: + app: my-coherence-app + version: 1.0.0 + serviceName: storage-headless # <1> + replicas: 3 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app: my-coherence-app + version: 1.0.0 + spec: + containers: + - name: coherence + image: ghcr.io/oracle/coherence-ce:22.06.6 # <2> + env: + - name: COHERENCE_CLUSTER # <3> + value: "test-cluster" + - name: NAMESPACE # <4> + valueFrom: + fieldRef: + fieldPath: "metadata.namespace" + - name: COHERENCE_WKA # <5> + value: "storage-wka.${NAMESPACE}.svc" + - name: COHERENCE_LOCALPORT # <6> + value: "7575" + - name: COHERENCE_LOCALHOST # <7> + valueFrom: + fieldRef: + fieldPath: "metadata.name" + - name: COHERENCE_MACHINE # <8> + valueFrom: + fieldRef: + fieldPath: "spec.nodeName" + - name: COHERENCE_MEMBER # <9> + valueFrom: + fieldRef: + fieldPath: "metadata.name" + ports: + - name: coherence # <10> + containerPort: 7574 + - name: coh-local + containerPort: 7575 + - name: extend-proxy + containerPort: 20000 + - name: grpc-proxy + containerPort: 1408 + - name: management + containerPort: 30000 + - name: metrics + containerPort: 9162 + readinessProbe: # <11> + httpGet: + path: "/ready" + port: 6676 + scheme: "HTTP" + livenessProbe: + httpGet: + path: "/healthz" + port: 6676 + scheme: "HTTP" +---- + +<1> All StatefulSets require a headless service, in this case the service will be named `storage-headless` to match the +service above +<2> This example is using the CE 22.06 image +<3> The `COHERENCE_CLUSTER` environment variable sets the Coherence cluster name to `test-cluster` +<4> The `NAMESPACE` environment variable contains the namespace the StatefulSet is deployed into. +The value is taken from the `matadata.namespace` field of the Pod. This is then used to create a fully qualified +well known address value +<5> The `COHERENCE_WKA` environment variable sets address Coherence uses to perform a DNS lookup for cluster member IP +addresses. In this case we use the name of the WKA service created above combined with the `NAMESPACE` environment +variable to give a fully qualified service name. +<6> The `COHERENCE_LOCALPORT` environment variable sets the Coherence localport to 7575, which matches what was exposed +in the Service ports and container ports +<7> The `COHERENCE_LOCAHOST` environment variable sets the hostname that Coherence binds to, in this case it will be +the same as the Pod name by using the "valueFrom" setting to get the value from the Pod's `metadata.name` field +<8> It is best practice to use the `COHERENCE_MACHINE` environment variable to set the Coherence machine label to the +Kubernetes Node name. The machine name is used by Coherence when assigning backup partitions, so a backup of a partition will +not be on the same Node as the primary owner of the partition. +the same as the Pod name by using the "valueFrom" setting to get the value from the Pod's `metadata.name` field +<9> It is best practice to use the `COHERENCE_MEMBER` environment variable to set the Coherence member name to the +Pod name. +<10> All the ports required are exposed as container ports. The names must correspond to the names used for the container ports in the Service spec. +<11> As we are using Coherence CE 22.06 we can use Coherence built in health check endpoints for the readiness and liveness probes. + + +=== Deploy the Cluster + +We will deploy the cluster into a Kubernetes namespace names `coherence`. +Before deploying the cluster we need to ensure it has been labeled so that Istio will inject the +Envoy proxy sidecar into the Pods. + +[source,bash] +---- +kubectl create namespace coherence +kubectl label namespace coherence istio-injection=enabled +---- + +To deploy the cluster we just apply all three yaml files to Kubernetes. +We could combine them into a single yaml file if we wanted to. + +[source,bash] +---- +kubectl apply -f wka-service.yaml +kubectl apply -f storage-service.yaml +kubectl apply -f storage.yaml +---- + +If we list the services, we see the two services we created + +[source,bash] +---- +$ kubectl get svc +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +storage-headless ClusterIP None 7574/TCP,7575/TCP,20000/TCP,1408/TCP,30000/TCP,9612/TCP 37m +storage-wka ClusterIP None 7574/TCP 16m +---- + +If we list the Pods, we see three Pods, as the StatefulSet replicas field is set to three. + +[source,bash] +---- +$ kubectl get pod +NAME READY STATUS RESTARTS AGE +storage-0 2/2 Running 0 7m47s +storage-1 2/2 Running 0 7m47s +storage-2 2/2 Running 0 7m47s +---- + +We can use Istio's Kiali dashboard to visualize the cluster we created. + +We labelled the resources with the `app` label with a value of `my-coherence-app` and we can see this application +in the Kiali dashboard. The graph shows the cluster member Pods communicating with each other via the `storage-headless` +service. The padlock icons show that this traffic is using mTLS even though Coherence has not been configured with TLS, +this is being provided by Istio. + +image::images/kiali-cluster-start.png[width=1024,height=512] + +== Coherence Clients + +Coherence clients (Extend or gRPC) can be configured to connect to the Coherence cluster. + +If the clients are also inside the cluster they can be configured to connect using the StatefulSet as the hostname +for the endpoints. Clients inside Kubernetes can also use the minimal Coherence NameService configuration where the +StatefulSet service name is used as the client's WKA address and the same cluster name is configured. + +Clients external to the Kubernetes cluster can be configured using any of the ingress or gateway features of Istio and Kubernetes. +All the different ways to do this are beyond the scope of this simple example as there are many, and they +depend on the versions of Istio and Kubernetes being used. + +When connecting Coherence Extend or gRPC clients from outside Kubernetes, the Coherence NameService cannot be used +by clients to look up the endpoints. The clients must be configured with fixed endpoints using the hostnames and ports +of the configured ingress or gateway services. + diff --git a/examples/no-operator/04_istio/images/kiali-cluster-start.png b/examples/no-operator/04_istio/images/kiali-cluster-start.png new file mode 100644 index 000000000..42c397f7d Binary files /dev/null and b/examples/no-operator/04_istio/images/kiali-cluster-start.png differ diff --git a/examples/no-operator/04_istio/storage-service.yaml b/examples/no-operator/04_istio/storage-service.yaml new file mode 100644 index 000000000..a83f51540 --- /dev/null +++ b/examples/no-operator/04_istio/storage-service.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Service +metadata: + name: storage-headless +spec: + clusterIP: None + selector: + app: my-coherence-app + version: 1.0.0 + ports: + - name: coherence + port: 7574 + targetPort: coherence + appProtocol: tcp + - name: coh-local + port: 7575 + targetPort: coh-local + appProtocol: tcp + - name: extend-proxy + port: 20000 + targetPort: extend-proxy + appProtocol: tcp + - name: grpc-proxy + port: 1408 + targetPort: grpc-proxy + appProtocol: grpc + - name: management + port: 30000 + targetPort: management + appProtocol: http + - name: metrics + port: 9612 + targetPort: metrics + appProtocol: http diff --git a/examples/no-operator/04_istio/storage.yaml b/examples/no-operator/04_istio/storage.yaml new file mode 100644 index 000000000..c3939fb9a --- /dev/null +++ b/examples/no-operator/04_istio/storage.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: storage + labels: + app: my-coherence-app + version: 1.0.0 +spec: + selector: + matchLabels: + app: my-coherence-app + version: 1.0.0 + serviceName: storage-headless # <1> + replicas: 3 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app: my-coherence-app + version: 1.0.0 + spec: + containers: + - name: coherence + image: ghcr.io/oracle/coherence-ce:22.06.6 # <2> + env: + - name: COHERENCE_CLUSTER # <3> + value: "test-cluster" + - name: NAMESPACE # <3> + valueFrom: + fieldRef: + fieldPath: "metadata.namespace" + - name: COHERENCE_WKA # <4> + value: "storage-wka.${NAMESPACE}.svc" + - name: COHERENCE_LOCALPORT # <5> + value: "7575" + - name: COHERENCE_LOCALHOST # <6> + valueFrom: + fieldRef: + fieldPath: "metadata.name" + - name: COHERENCE_MACHINE # <7> + valueFrom: + fieldRef: + fieldPath: "spec.nodeName" + - name: COHERENCE_MEMBER # <8> + valueFrom: + fieldRef: + fieldPath: "metadata.name" + ports: + - name: coherence # <9> + containerPort: 7574 + - name: coh-local + containerPort: 7575 + - name: extend-proxy + containerPort: 20000 + - name: grpc-proxy + containerPort: 1408 + - name: management + containerPort: 30000 + - name: metrics + containerPort: 9162 + readinessProbe: # <10> + httpGet: + path: "/ready" + port: 6676 + scheme: "HTTP" + livenessProbe: + httpGet: + path: "/healthz" + port: 6676 + scheme: "HTTP" \ No newline at end of file diff --git a/examples/no-operator/04_istio/wka-service.yaml b/examples/no-operator/04_istio/wka-service.yaml new file mode 100644 index 000000000..fb49da69c --- /dev/null +++ b/examples/no-operator/04_istio/wka-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: storage-wka +spec: + clusterIP: None + publishNotReadyAddresses: true + selector: + app: my-coherence-app + version: 1.0.0 + ports: + - name: coherence + port: 7574 + targetPort: coherence + appProtocol: tcp diff --git a/examples/no-operator/README.adoc b/examples/no-operator/README.adoc index 081033928..9a6c7b701 100644 --- a/examples/no-operator/README.adoc +++ b/examples/no-operator/README.adoc @@ -65,4 +65,11 @@ Expands the simple storage enabled server to expose metrics that can be scraped -- Expands the simple storage enabled server to secure Extend using TLS. -- + +[CARD] +.Running Coherence with Istio +[link=examples/no-operator/04_istio/README.adoc] +-- +Expands the simple storage enabled server to secure Extend using TLS. +-- ==== diff --git a/examples/no-operator/test-client/pom.xml b/examples/no-operator/test-client/pom.xml index 4deba7220..8ac7c80e8 100644 --- a/examples/no-operator/test-client/pom.xml +++ b/examples/no-operator/test-client/pom.xml @@ -23,7 +23,7 @@ 11 - 22.06.4 + 22.06.6 ${project.basedir} diff --git a/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/Duration.java b/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/Duration.java new file mode 100644 index 000000000..a9f6bb2e2 --- /dev/null +++ b/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/Duration.java @@ -0,0 +1,418 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package com.oracle.coherence.k8s; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A {@link Duration} represents an amount of time, with nanosecond accuracy. + */ +public class Duration { + /** + * Construct a {@link Duration} by parsing the specified {@link String}. + *

+ * The format of the {@link String} consists of one or more numbers, each with a specific {@link Magnitude} + * separated by white space. + *

+ * Note 1: Numbers may contain decimal places. + *

+ * Note 2: Each number must be followed by a specific {@link Magnitude}. For a more relaxed {@link String}-based + * constructor, including the ability to specify a default {@link Magnitude}, use + * {@link #Duration(String, Magnitude)} instead. + *

+ * For example: The following are valid {@link Duration}s. + * "10s", "0ns", "1hr 10m", "5us", "100ms", "12m", "1.5h", "20m 10.5s" + * + * @param s the string containing the {@link Duration} + */ + public Duration(String s) { + this(s, null); + } + + /** + * Construct a {@link Duration} by parsing the specified {@link String}. + *

+ * The format of the {@link String} is either a number (without a specified {@link Magnitude}) or a {@link String} + * containing one or more numbers, each with a specific {@link Magnitude} separated by white space. + *

+ * Note: numbers may contain decimal places. + *

+ * For example: The following are valid {@link Duration}s. + * "10s", "0ns", "1hr 10m", "0", "5us", "100ms", "12m", "1.5h", "20m 10.5s" + * + * @param s the string containing the {@link Duration} + * @param m the default {@link Magnitude} to use if the specified {@link String} does not specify a + * {@link Magnitude}. when null a {@link Magnitude} must be specified in the {@link String}. + * when not null the {@link String} may only contain a single number. + */ + public Duration(String s, Magnitude m) { + s = (s == null) ? null : s.trim(); + + if ((s == null) || s.isEmpty()) { + throw new IllegalArgumentException("An empty or null string was provided. Expected a duration size"); + } + + // when a default magnitude was specified, attempt to match just the number + boolean fUsedDefault = false; + + if (m != null) { + Matcher matcher = REGEX_NUMBER.matcher(s); + + if (matcher.matches()) { + String sAmount = matcher.group(1); + double nAmount = Double.parseDouble(sAmount); + + durationNanos = Math.round(nAmount * m.getFactor()); + fUsedDefault = true; + } + } + + // when a default magnitude wasn't used, attempt to match using explicit magnitudes + if (!fUsedDefault && !s.equals("0")) { + Matcher matcher = REGEX_PATTERN.matcher(s); + + if (!matcher.matches()) { + throw new IllegalArgumentException(String.format("The specified %s [%s] is invalid.", + this.getClass().getName(), s)); + } + + durationNanos = 0; + + int i = 1; + + while (i < matcher.groupCount()) { + String sAmount = matcher.group(i + 1); + + if (sAmount != null) { + // determine the amount of the magnitude + double nAmount = Double.parseDouble(sAmount); + + // determine the magnitude + String sSuffix = matcher.group(i + 2); + Magnitude magnitude = Magnitude.fromSuffix(sSuffix); + + durationNanos += Math.round(nAmount * magnitude.getFactor()); + } + + i += 3; + } + } + } + + // ----- Duration methods ----------------------------------------------- + + /** + * Obtains the number of nanoseconds in the {@link Duration}. + * + * @return The number of nanoseconds + */ + public long getNanos() { + return durationNanos; + } + + /** + * Obtains the {@link Duration} in the specified {@link Magnitude} (rounded down). + * + * @param magnitude the required {@link Magnitude} + * @return The number of units of the specified {@link Magnitude}. + */ + public long as(Magnitude magnitude) { + return durationNanos / magnitude.getFactor(); + } + + /** + * Obtains a {@link String} representation of the {@link Duration} using the most appropriate {@link Magnitude} + * to simplify the representation. + *

+ * Note: Using {@link #toString()} will result in a non-exact representation. + * + * @param fExact indicates an exact value is required or if a rounded value will suffice. + * @return A {@link String} + */ + public String toString(boolean fExact) { + StringBuilder sbResult = new StringBuilder(); + long cRemainingNanos = durationNanos; + int cLimit = 2; + + for (Magnitude magnitude = Magnitude.HIGHEST; cRemainingNanos > 0 && (fExact || cLimit > 0); + magnitude = magnitude.previous()) { + long cMagnitudeUnits = cRemainingNanos / magnitude.getFactor(); + + if (cMagnitudeUnits > 0) { + cRemainingNanos -= cMagnitudeUnits * magnitude.getFactor(); + + if (fExact || magnitude.ordinal() > Magnitude.SECOND.ordinal()) { + sbResult.append(cMagnitudeUnits); + } + else { + // non-exact and seconds or less, express as fraction + long cNanosPerMagnitudeFractionUnit = magnitude.getFactor() / 1000; + long cMagnitudeFractionUnits = (cNanosPerMagnitudeFractionUnit > 0) + ? cRemainingNanos / cNanosPerMagnitudeFractionUnit : 0; + long cMagnitudeFractionNanos = cMagnitudeFractionUnits * cNanosPerMagnitudeFractionUnit; + + double flRem = cMagnitudeFractionNanos * 1000 / magnitude.getFactor() / 1000.0; + + if (flRem >= 0.01 && (fExact || cLimit > 1)) { + sbResult.append(String.format("%.2f", cMagnitudeUnits + flRem)); + } + else { + sbResult.append(cMagnitudeUnits); + } + cRemainingNanos = 0; + } + + sbResult.append(magnitude.getSuffix()); + --cLimit; + } + else if (sbResult.length() > 0) { + --cLimit; + } + } + + if (sbResult.length() == 0) { + return "0ns"; + } + else { + return sbResult.toString(); + } + } + + /** + * Return this {@link Duration} as a {@link java.time.Duration Java Time Duration}. + * + * @return this {@link Duration} as a {@link java.time.Duration Java Time Duration} + */ + public java.time.Duration asJavaDuration() { + return java.time.Duration.ofNanos(getNanos()); + } + + // ----- Object methods ------------------------------------------------- + + /** + * {@inheritDoc} + */ + @Override + public int hashCode() { + return 31 + (int) (durationNanos ^ (durationNanos >>> 32)); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean equals(Object obj) { + return (this == obj) || ((obj != null) && (obj instanceof Duration) && ((Duration) obj).durationNanos == durationNanos); + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + return toString(false); + } + + // ----- Magnitude Enumeration ------------------------------------------ + + /** + * The {@link Magnitude} of the {@link Duration}. + */ + public enum Magnitude { + /** + * The nanosecond magnitude. + */ + NANO(1L, "ns"), + /** + * The microsecond magnitude. + */ + MICRO(1000L, "us", "\u00B5s", "\u039Cs", "\u03BCs"), + /** + * The millisecond magnitude. + */ + MILLI(1000000L, "ms"), + /** + * The second magnitude. + */ + SECOND(1000000000L, "s"), + /** + * The minute magnitude. + */ + MINUTE(60 * 1000000000L, "m"), + /** + * The hour magnitude. + */ + HOUR(60 * 60 * 1000000000L, "h"), + /** + * The day magnitude. + */ + DAY(24 * 60 * 60 * 1000000000L, "d"); + + // ----- constructors ----------------------------------------------- + + /** + * Construct a {@link Magnitude}. + * + * @param nFactor the factor which this Magnitude represents, relative to the number of nanoseconds + * @param sSuffixes the suffix (case-insensitive) of the magnitude + */ + Magnitude(long nFactor, String... sSuffixes) { + factor = nFactor; + suffixes = sSuffixes; + } + + // ----- Magnitude methods ------------------------------------------ + + /** + * Determine the factor of the {@link Magnitude} relative to the number of + * nanoseconds. For example, "MILLI" has a factor of 1000000. + * + * @return the factor of the {@link Magnitude} + */ + public long getFactor() { + return factor; + } + + /** + * Obtain the default for the {@link Magnitude}. For example, "MILLI" has the suffix + * "ms". + * + * @return the suffix of the {@link Magnitude}. + */ + public String getSuffix() { + return suffixes[0]; + } + + /** + * Determine if the passed suffix is compatible with this {@link Magnitude}'s suffix, ignoring case. + * + * @param s the suffix to test + * @return true iff the passed string is compatible with the suffix of this {@link Magnitude}. + */ + public boolean isSuffix(String s) { + for (String sSuffix : suffixes) { + if (sSuffix.equalsIgnoreCase(s)) { + return true; + } + } + + return false; + } + + /** + * Obtain the next order of {@link Magnitude} (above this one). + * + * @return the next order of {@link Magnitude} above this one or null if this is + * the {@link #HIGHEST}. + */ + public Magnitude next() { + if (this.equals(Magnitude.HIGHEST)) { + return null; + } + else { + return Magnitude.VALUES[this.ordinal() + 1]; + } + } + + /** + * Obtain the previous order of {@link Magnitude} (above this one). + * + * @return the previous order of {@link Magnitude} or null if this is + * the {@link #LOWEST}. + */ + public Magnitude previous() { + if (this.equals(Magnitude.LOWEST)) { + return null; + } + else { + return Magnitude.VALUES[this.ordinal() - 1]; + } + } + + // ----- helpers ---------------------------------------------------- + + /** + * Determine the {@link Magnitude} given the specified suffix. + * + * @param sSuffix the proposed suffix + * @return a {@link Magnitude} with the specified suffix + */ + public static Magnitude fromSuffix(String sSuffix) { + sSuffix = sSuffix.trim(); + + if (sSuffix.length() == 0) { + return Magnitude.NANO; + } + else if (sSuffix.length() > 0) { + for (Magnitude magnitude : Magnitude.VALUES) { + if (magnitude.isSuffix(sSuffix)) { + return magnitude; + } + } + } + + throw new IllegalArgumentException(String.format("Unknown %s suffix [%s]", Magnitude.class.getName(), + sSuffix)); + } + + // ----- constants -------------------------------------------------- + + /** + * Cached copy of the VALUES array to avoid garbage creation. + */ + private static final Magnitude[] VALUES = Magnitude.values(); + + /** + * The lowest defined order of {@link Magnitude}. + */ + public static final Magnitude LOWEST = Magnitude.VALUES[0]; + + /** + * The highest defined order of {@link Magnitude}. + */ + public static final Magnitude HIGHEST = Magnitude.VALUES[Magnitude.VALUES.length - 1]; + + // ----- data members ----------------------------------------------- + + /** + * The number of nanoseconds in a single unit of this magnitude. For + * example, a minute has 60 billion nanoseconds. + */ + private final long factor; + + /** + * The suffixes that for the {@link Magnitude}. For example, "MILLI" has + * the suffix "ms". + */ + private final String[] suffixes; + } + + // ----- constants ------------------------------------------------------ + + /** + * The pre-compiled regular expression {@link Pattern}s to match + * a {@link Duration} specified as a {@link String}. + */ + private static final String NUMBER = "(\\d+(?:\\.\\d+)?)"; + private static final String PATTERN_MINUTE = "\\s*(" + NUMBER + "\\s*([Mm]))?"; + private static final String PATTERN_HOUR = "\\s*(" + NUMBER + "\\s*([Hh]))?"; + private static final String PATTERN_DAY = "\\s*(" + NUMBER + "\\s*([Dd]))?"; + private static final String PATTERN_SECOND = "\\s*(" + NUMBER + "\\s*([Ss]))?"; + private static final String PATTERN_NANO = "\\s*(" + NUMBER + "\\s*([Nn][Ss]))?"; + private static final String PATTERN_MILLI = "\\s*(" + NUMBER + "\\s*([Mm][Ss]))?"; + private static final String PATTERN_MICRO = "\\s*(" + NUMBER + "\\s*([Uu\u00B5Mm\\u039C\\u03BC][Ss]))?"; + private static final Pattern REGEX_NUMBER = Pattern.compile(NUMBER); + private static final Pattern REGEX_PATTERN = Pattern.compile(PATTERN_DAY + PATTERN_HOUR + PATTERN_MINUTE + + PATTERN_SECOND + PATTERN_MILLI + PATTERN_MICRO + PATTERN_NANO); + + // ----- data members --------------------------------------------------- + + /** + * The number of nanos in the {@link Duration}. + */ + private long durationNanos; +} diff --git a/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/OperatorRestServer.java b/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/OperatorRestServer.java index 8c0f09d48..de2172a6f 100644 --- a/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/OperatorRestServer.java +++ b/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/OperatorRestServer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2022, Oracle and/or its affiliates. + * Copyright (c) 2019, 2023, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -654,7 +654,7 @@ private void handleError(HttpExchange t, Throwable thrown, String action) { * * @return {@code true} if the Coherence cluster has members */ - private boolean hasClusterMembers() { + protected boolean hasClusterMembers() { Cluster cluster = clusterSupplier.get(); return cluster != null && cluster.isRunning() && !cluster.getMemberSet().isEmpty(); } diff --git a/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/Sleep.java b/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/Sleep.java new file mode 100644 index 000000000..83c690de5 --- /dev/null +++ b/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/Sleep.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package com.oracle.coherence.k8s; + +import java.util.Properties; +import java.util.Set; + +import com.tangosol.net.CacheFactory; +import com.tangosol.net.Cluster; + +/** + * A main class that sleeps for a specific duration. + */ +public class Sleep { + /** + * The default number of milliseconds to sleep. + */ + public static final long DEFAULT_SLEEP = 60000; + + /** + * Private constructor for utility class. + */ + private Sleep() { + } + + /** + * Sleep main entry point. + * + * @param args the arguments + * + * @throws Exception if an error occurs + */ + public static void main(String[] args) throws Exception { + + Properties props = new Properties(); + try (ReadinessServer rest = new ReadinessServer()) { + rest.start(); + + long cMillis; + if (args.length == 0) { + cMillis = DEFAULT_SLEEP; + } + else { + cMillis = new Duration(args[0]).asJavaDuration().toMillis(); + } + + cMillis = Math.max(10000, cMillis); + + long cSeconds = cMillis / 1000; + CacheFactory.log("Sleeping for " + cSeconds + " seconds"); + Thread.sleep(cMillis); + } + } + + /** + * A simple readiness probe server. + */ + private static class ReadinessServer + extends OperatorRestServer { + + /** + * Create a {@link ReadinessServer}. + */ + private ReadinessServer() { + super(() -> null, () -> {}, new Properties()); + } + + @Override + protected boolean hasClusterMembers() { + return true; + } + + @Override + boolean isStatusHA() { + return true; + } + + @Override + boolean isStatusHA(String exclusions) { + return true; + } + + @Override + boolean areCacheServicesHA(Cluster cluster, Set allowEndangered) { + return true; + } + + @Override + boolean isPersistenceIdle() { + return true; + } + } +} diff --git a/java/operator-test-client/src/main/java/com/oracle/coherence/k8s/testing/ExtendClient.java b/java/operator-test-client/src/main/java/com/oracle/coherence/k8s/testing/ExtendClient.java index 417632086..50a5e2fe6 100644 --- a/java/operator-test-client/src/main/java/com/oracle/coherence/k8s/testing/ExtendClient.java +++ b/java/operator-test-client/src/main/java/com/oracle/coherence/k8s/testing/ExtendClient.java @@ -1,11 +1,12 @@ /* - * Copyright (c) 2021, Oracle and/or its affiliates. + * Copyright (c) 2021, 2023, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ package com.oracle.coherence.k8s.testing; +import com.tangosol.coherence.config.Config; import com.tangosol.net.CacheFactory; import com.tangosol.net.NamedCache; @@ -32,6 +33,16 @@ public static void main(String[] args) { System.out.println("Putting key and value into cache 'test'"); cache.put("key-1", "value-1"); + int cIter = Config.getInteger("coherence.client.iterations", 0); + int cMillis = Config.getInteger("coherence.client.wait", 1000); + for (int i = 0; i < cIter; i++) { + System.out.println("Test iteration " + i); + cache.put("key-1", "value-1"); + Thread.sleep(cMillis); + cache.get("key-1"); + Thread.sleep(cMillis); + } + System.out.println("Test completed successfully"); System.exit(0); } diff --git a/java/pom.xml b/java/pom.xml index 4ae30c906..0401a8f09 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -65,7 +65,7 @@ 4.13.1 5.7.2 3.10.0 - 2.7.7 + 2.7.17 1.8.4 2.8.9 diff --git a/pkg/probe/probe.go b/pkg/probe/probe.go index db465b5ce..f9b48ff8b 100644 --- a/pkg/probe/probe.go +++ b/pkg/probe/probe.go @@ -55,7 +55,7 @@ func (in *CoherenceProbe) SetGetPodHostName(fn func(pod corev1.Pod) string) { func (in *CoherenceProbe) GetPodHostName(pod corev1.Pod) string { if in.getPodHostName == nil { - return pod.Status.PodIP + return pod.Spec.Hostname + "." + pod.Spec.Subdomain + "." + pod.GetNamespace() + ".svc" } return in.getPodHostName(pod) } diff --git a/pkg/runner/cmd_jshell.go b/pkg/runner/cmd_jshell.go new file mode 100644 index 000000000..46b5e48e5 --- /dev/null +++ b/pkg/runner/cmd_jshell.go @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ + +package runner + +import ( + v1 "github.com/oracle/coherence-operator/api/v1" + "github.com/spf13/cobra" + "os" +) + +const ( + // CommandJShell is the argument to launch a JShell console. + CommandJShell = "jshell" +) + +// queryPlusCommand creates the corba "jshell" sub-command +func jShellCommand() *cobra.Command { + return &cobra.Command{ + Use: CommandJShell, + Short: "Start a Coherence interactive JShell console", + Long: "Starts a Coherence interactive JShell console", + Args: cobra.ArbitraryArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return run(cmd, jShell) + }, + } +} + +// Configure the runner to run a Coherence JShell console +func jShell(details *RunDetails, _ *cobra.Command) { + details.Command = CommandQueryPlus + details.AppType = AppTypeJava + details.MainClass = "jdk.internal.jshell.tool.JShellToolProvider" + if len(os.Args) > 2 { + details.MainArgs = os.Args[2:] + } + details.addArg("-Dcoherence.distributed.localstorage=false") + details.setenv(v1.EnvVarCohRole, "jshell") + details.unsetenv(v1.EnvVarJvmMemoryHeap) +} diff --git a/pkg/runner/cmd_sleep.go b/pkg/runner/cmd_sleep.go new file mode 100644 index 000000000..72f77d1ae --- /dev/null +++ b/pkg/runner/cmd_sleep.go @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ + +package runner + +import ( + v1 "github.com/oracle/coherence-operator/api/v1" + "github.com/spf13/cobra" + "os" +) + +const ( + // CommandSleep is the argument to sleep for a number of seconds. + CommandSleep = "sleep" +) + +// queryPlusCommand creates the corba "sleep" sub-command +func sleepCommand() *cobra.Command { + return &cobra.Command{ + Use: CommandSleep, + Short: "Sleep for a number of seconds", + Long: "Sleep for a number of seconds", + Args: cobra.ArbitraryArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return run(cmd, sleep) + }, + } +} + +func sleep(details *RunDetails, _ *cobra.Command) { + details.Command = CommandSleep + details.AppType = AppTypeJava + details.MainClass = "com.oracle.coherence.k8s.Sleep" + if len(os.Args) > 2 { + details.MainArgs = os.Args[2:] + } + details.UseOperatorHealth = true + details.addArg("-Dcoherence.distributed.localstorage=false") + details.setenv(v1.EnvVarCohRole, "sleep") + details.unsetenv(v1.EnvVarJvmMemoryHeap) +} diff --git a/pkg/runner/run_details.go b/pkg/runner/run_details.go index 1a51962f6..86d05bd37 100644 --- a/pkg/runner/run_details.go +++ b/pkg/runner/run_details.go @@ -39,19 +39,20 @@ func NewRunDetails(v *viper.Viper) *RunDetails { // RunDetails contains the information to run an application. type RunDetails struct { - Command string - CoherenceHome string - JavaHome string - UtilsDir string - Dir string - GetSite bool - AppType string - Classpath string - Args []string - MainClass string - MainArgs []string - BuildPacks *bool - env *viper.Viper + Command string + CoherenceHome string + JavaHome string + UtilsDir string + Dir string + GetSite bool + UseOperatorHealth bool + AppType string + Classpath string + Args []string + MainClass string + MainArgs []string + BuildPacks *bool + env *viper.Viper } // Getenv returns the value for the specified environment variable, or empty string if not set. diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 3b9677c70..edd666a8b 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -50,6 +50,8 @@ const ( AppTypeHelidon = "helidon" // AppTypeSpring is the argument to specify a Spring application. AppTypeSpring = "spring" + // AppTypeOperator is the argument to specify running an Operator command. + AppTypeOperator = "operator" // defaultConfig is the root name of the default configuration file defaultConfig = ".coherence-runner" @@ -123,6 +125,8 @@ func NewRootCommand(env map[string]string) (*cobra.Command, *viper.Viper) { rootCmd.AddCommand(nodeCommand()) rootCmd.AddCommand(operatorCommand()) rootCmd.AddCommand(networkTestCommand()) + rootCmd.AddCommand(jShellCommand()) + rootCmd.AddCommand(sleepCommand()) return rootCmd, v } @@ -592,6 +596,9 @@ func createCommand(details *RunDetails) (string, *exec.Cmd, error) { case details.AppType == AppTypeCoherence: app = "Java" cmd, err = createJavaCommand(details.getJavaExecutable(), details) + case details.AppType == AppTypeOperator: + app = "Operator" + cmd, err = createOperatorCommand(details) default: app = "Graal (" + details.AppType + ")" cmd, err = createGraalCommand(details) @@ -650,6 +657,25 @@ func _createJavaCommand(javaCmd string, details *RunDetails, args []string) (*ex return cmd, nil } +func createOperatorCommand(details *RunDetails) (*exec.Cmd, error) { + executable := os.Args[0] + args := details.MainArgs[1:] + cmd := exec.Command(executable, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + + if details.Dir != "" { + _, err := os.Stat(details.Dir) + if err != nil { + return nil, errors.Wrapf(err, "Working directory %s does not exists or is not a directory", details.Dir) + } + cmd.Dir = details.Dir + } + + return cmd, nil +} + func _createBuildPackCommand(_ *RunDetails, className string, args []string) (*exec.Cmd, error) { launcher := getBuildpackLauncher() @@ -912,12 +938,16 @@ func cohPost12214(details *RunDetails) { } func cohPost2206(details *RunDetails) { - useOperator, found := os.LookupEnv(v1.EnvVarUseOperatorHealthCheck) - if found && strings.EqualFold("true", useOperator) { + if details.UseOperatorHealth { details.addArg("-Dcoherence.k8s.operator.health.enabled=true") } else { - details.addArg("-Dcoherence.k8s.operator.health.enabled=false") - details.setSystemPropertyFromEnvVarOrDefault(v1.EnvVarCohHealthPort, "-Dcoherence.health.http.port", fmt.Sprintf("%d", v1.DefaultHealthPort)) + useOperator := details.getenvOrDefault(v1.EnvVarUseOperatorHealthCheck, "false") + if strings.EqualFold("true", useOperator) { + details.addArg("-Dcoherence.k8s.operator.health.enabled=true") + } else { + details.addArg("-Dcoherence.k8s.operator.health.enabled=false") + details.setSystemPropertyFromEnvVarOrDefault(v1.EnvVarCohHealthPort, "-Dcoherence.health.http.port", fmt.Sprintf("%d", v1.DefaultHealthPort)) + } } } diff --git a/test/e2e/clients/storage.yaml b/test/e2e/clients/storage.yaml index c498b3e3e..2f08055d7 100644 --- a/test/e2e/clients/storage.yaml +++ b/test/e2e/clients/storage.yaml @@ -3,11 +3,6 @@ kind: Coherence metadata: name: storage spec: - annotations: - traffic.sidecar.istio.io/excludeInboundPorts: "7574,7575,7576" - coherence: - localPort: 7575 - localPortAdjust: 7576 ports: - name: extend port: 20000 diff --git a/test/e2e/compatibility/compatibility_test.go b/test/e2e/compatibility/compatibility_test.go index eca3f4da2..3984553a5 100644 --- a/test/e2e/compatibility/compatibility_test.go +++ b/test/e2e/compatibility/compatibility_test.go @@ -61,8 +61,9 @@ func TestCompatibility(t *testing.T) { UpgradeToCurrentVersion(t, g, ns, name) // wait a few minutes to allow the new Operator to reconcile the existing Coherence cluster + // usually this would be quick, but on a slow build machine it could be a few minutes t.Logf("Upgraded to current Operator version - waiting for reconcile...\n") - time.Sleep(1 * time.Minute) + time.Sleep(5 * time.Minute) // Get the current state of the StatefulSet stsAfter := &appsv1.StatefulSet{} diff --git a/test/e2e/helper/port_forward.go b/test/e2e/helper/port_forward.go index 642bf11e6..dd21d2f6d 100644 --- a/test/e2e/helper/port_forward.go +++ b/test/e2e/helper/port_forward.go @@ -49,6 +49,7 @@ type PortForwarder struct { Running bool stopChan chan struct{} + forwarder *portforward.PortForwarder lock sync.Mutex KubeClient kubernetes.Interface } @@ -148,7 +149,7 @@ func (f *PortForwarder) Start() error { out, errOut := new(bytes.Buffer), new(bytes.Buffer) - forwarder, err := portforward.New(dialer, f.Ports, f.stopChan, readyChan, out, errOut) + f.forwarder, err = portforward.New(dialer, f.Ports, f.stopChan, readyChan, out, errOut) if err != nil { return err } @@ -162,7 +163,7 @@ func (f *PortForwarder) Start() error { }() go func() { - if err = forwarder.ForwardPorts(); err != nil { // Locks until stopChan is closed. + if err = f.forwarder.ForwardPorts(); err != nil { // Locks until stopChan is closed. pfError = err fmt.Println(err) close(readyChan) @@ -170,7 +171,7 @@ func (f *PortForwarder) Start() error { }() // blocks until forwarder is ready - <-forwarder.Ready + <-f.forwarder.Ready f.Running = true