Fix broker hostname in function / proxy-ws configuration (TLS enabled) (

#197)

* Replace `brokerSts.component` -> `brokerSts.component` (function)

* Replace `brokerSts.component` -> `brokerSts.component` (proxy)

* pulsar+ssl when using TLS port in function configmap

Co-authored-by: Michael Marshall <[email protected]>

* Add `useStsBrokersForDiscovery` and document to values

Co-authored-by: Michael Marshall <[email protected]>

This attempts to achieve:

a. fix #196
b. improve the broker hostname in the proxy-ws configuration 

Note: b. doesn't seem to actually fix anything, but I believe it's a consistent change to make if we intend to use brokerSts, whenever enabling TLS.

Both changes have been tested with TLS enabled in the following scenario:


1. Deploy pulsar with tls and functions enabled (values below)
2. create a partitioned topic and subscription
3. generate a source function:
```
                        bin/pulsar-admin sources create
                        -t data-generator --name data-generator-source
                        --source-config '{"sleepBetweenMessages":"10"}'
                        --destination-topic-name persistent://public/default/test
```
4. consume messages

Values:

```
enableAntiAffinity: no
enableTls: yes
tls:
  function:
    enableTlsWithBroker: true
    enableHostnameVerification: true
cert-manager:
  enabled: true
createCertificates:
  selfSigned:
    enabled: true
enableTokenAuth: yes
autoRecovery:
  enableProvisionContainer: yes
restartOnConfigMapChange:
  enabled: yes
image:
  zookeeper:
    repository: datastaxlunastreaming-all
    tag: 2.7.2_1.1.32
  bookie:
    repository: datastaxlunastreaming-all
    tag: 2.7.2_1.1.32
  bookkeeper:
    repository: datastaxlunastreaming-all
    tag: 2.7.2_1.1.32
  autorecovery:
    repository: datastaxlunastreaming-all
    tag: 2.7.2_1.1.32
  broker:
    repository: datastaxlunastreaming-all
    tag: 2.7.2_1.1.32
  proxy:
    repository: datastaxlunastreaming-all
    tag: 2.7.2_1.1.32
  functions:
    repository: datastaxlunastreaming-all
    tag: 2.7.2_1.1.32
  function:
    repository: datastaxlunastreaming-all
    tag: 2.7.2_1.1.32
extra:
  broker: false
  brokerSts: true
  function: yes
  burnell: yes
  burnellLogCollector: yes
  pulsarHeartbeat: yes
  pulsarAdminConsole: yes
  functionsAsPods: yes
default_storage:
  existingStorageClassName: server-storage
volumes:
  data: #ASF Helm Chart
    storageClassName: existent-storage-class
zookeeper:
  replicaCount: 3
bookkeeper:
  replicaCount: 3
broker:
  component: broker
  replicaCount: 2
  ledger:
    defaultEnsembleSize: 1
    defaultAckQuorum:  1
    defaultWriteQuorum: 1
function:
  replicaCount: 1
  functionReplicaCount: 1
  runtime: "kubernetes"
proxy:
  disableZookeeperDiscovery: true
  useStsBrokersForDiscovery: true
  replicaCount: 2
  autoPortAssign:
    enablePlainTextWithTLS: yes
  service:
    type: ClusterIP
    autoPortAssign:
      enabled: yes
grafanaDashboards:
  enabled: yes
pulsarAdminConsole:
  replicaCount: 0
  service:
    type: ClusterIP
grafana: #ASF Helm Chart
  service:
    type: ClusterIP
pulsar_manager:
  service: #ASF Helm Chart
    type: ClusterIP
kube-prometheus-stack: # Luna Streaming Helm Chart
  enabled: no
  prometheusOperator:
    enabled: no
  grafana:
    enabled: no
    service:
      type: ClusterIP
pulsarSQL:
  service:
    type: ClusterIP
```

helm-chart-sources/pulsar/templates/function/function-configmap.yaml

@@ -164,9 +164,13 @@ data:
       # setting this to true is let function worker to submit functions to the same k8s cluster as function worker
       # is running. setting this to false if your function worker is not running as a k8 pod.
       submittingInsidePod: true
+      {{- if and .Values.enableTls .Values.tls.function.enableTlsWithBroker }}
       # setting the pulsar service url that pulsar function should use to connect to pulsar
       # if it is not set, it will use the pulsar service url configured in worker service
+      pulsarServiceUrl: "pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}.{{ template "pulsar.serviceDnsSuffix" . }}:6651/"
+      {{- else }}
       pulsarServiceUrl: "pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.serviceDnsSuffix" . }}:6650/"
+      {{- end }}
       # setting the pulsar admin url that pulsar function should use to connect to pulsar
       # if it is not set, it will use the pulsar admin url configured in worker service
       pulsarAdminUrl: "http://{{ template "pulsar.fullname" . }}-{{ .Values.function.component }}.{{ template "pulsar.serviceDnsSuffix" . }}:6750/"

helm-chart-sources/pulsar/templates/proxy/proxy-ws-configmap.yaml

@@ -30,11 +30,18 @@ metadata:
     cluster: {{ template "pulsar.fullname" . }}
 data:
   {{- if .Values.proxy.disableZookeeperDiscovery }}
+  {{- if .Values.proxy.useStsBrokersForDiscovery }}
+  brokerServiceURL: "pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}.{{ template "pulsar.serviceDnsSuffix" . }}:6650"
+  brokerServiceURLTLS: "pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}.{{ template "pulsar.serviceDnsSuffix" . }}:6651"
+  brokerWebServiceURL: "http://{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}.{{ template "pulsar.serviceDnsSuffix" . }}:8080"
+  brokerWebServiceURLTLS: "https://{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}.{{ template "pulsar.serviceDnsSuffix" . }}:8443"
+  {{- else }}
   brokerServiceUrl: "pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.serviceDnsSuffix" . }}:6650"
   brokerServiceUrlTls: "pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.serviceDnsSuffix" . }}:6651"
   serviceUrl: "http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.serviceDnsSuffix" . }}:8080"
   serviceUrlTls: "https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ template "pulsar.serviceDnsSuffix" . }}:8443"
   {{- end }}
+  {{- end }}
   zookeeperServers:
     {{- if and .Values.enableTls .Values.tls.zookeeper.enabled }}
     "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-ca.{{ template "pulsar.serviceDnsSuffix" . }}:2281"

helm-chart-sources/pulsar/values.yaml

@@ -1267,7 +1267,8 @@ proxy:
   component: proxy
   replicaCount: 3
   disableZookeeperDiscovery: true
-  useStsBrokersForDiscover: false
+  # use "brokersts" instead of "broker" to identify the broker k8s component
+  useStsBrokersForDiscovery: false
   updateStrategy:
     type: RollingUpdate
     rollingUpdate: