functions_worker.yml

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

####################
# Worker Settings
####################

workerId: standalone
workerHostname: localhost
workerPort: 6750
workerPortTls: 6751

# Configuration Store connection string
configurationStoreServers: localhost:2181
# ZooKeeper session timeout in milliseconds
zooKeeperSessionTimeoutMillis: 30000
# ZooKeeper operation timeout in seconds
zooKeeperOperationTimeoutSeconds: 30

################################
# Function package management
################################

numFunctionPackageReplicas: 1
downloadDirectory: download/pulsar_functions
# Classname of Pluggable JVM GC metrics logger that can log GC specific metrics
# jvmGCMetricsLoggerClassName:

#################################################################
# Function metadata managment (assignment, scheduling, and etc)
#################################################################

# Configure the pulsar client used by function metadata management
#
# points
# Whether to enable TLS when clients connect to broker
useTls: false
# For TLS:
# brokerServiceUrl=pulsar+ssl://localhost:6651/
pulsarServiceUrl: pulsar://localhost:6650
# For TLS:
# webServiceUrl=https://localhost:8443/
pulsarWebServiceUrl: http://localhost:8080

############################################
# security settings for pulsar broker client
############################################
# The path to trusted certificates used by the Pulsar client to authenticate with Pulsar brokers
# brokerClientTrustCertsFilePath:
# Whether to enable the broker client authentication used by function workers to talk to brokers
# brokerClientAuthenticationEnabled: false
# the authentication plugin to be used by the pulsar client used in worker service
# brokerClientAuthenticationPlugin:
# the authentication parameter to be used by the pulsar client used in worker service
# brokerClientAuthenticationParameters:

# Bookie Authentication
#
# Authentication plugin to use when connecting to bookies
# bookkeeperClientAuthenticationPlugin:
# BookKeeper auth plugin implementatation specifics parameters name and values
# bookkeeperClientAuthenticationParametersName:
# Parameters for bookkeeper auth plugin
# bookkeeperClientAuthenticationParameters:


# pulsar topics used for function metadata management

pulsarFunctionsNamespace: public/functions
pulsarFunctionsCluster: standalone
functionMetadataTopicName: metadata
clusterCoordinationTopicName: coordinate
# Should the metadata topic be compacted? Defaulting it to false for backwards compatibility
useCompactedMetadataTopic: false
# Number of threads to use for HTTP requests processing. Default is set to 8
numHttpServerThreads: 8

# function assignment and scheduler

schedulerClassName: "org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler"
functionAssignmentTopicName: "assignments"
failureCheckFreqMs: 30000
rescheduleTimeoutMs: 60000
# frequency at which to check if cluster needs rebalancing (set to -1 to disable)
rebalanceCheckFreqSec: -1
initialBrokerReconnectMaxRetries: 60
assignmentWriteMaxRetries: 60
instanceLivenessCheckFreqMs: 30000
# Frequency how often worker performs compaction on function-topics
topicCompactionFrequencySec: 1800
# should the function context have pulsar admin client exposed? Default is disabled.
exposeAdminClientEnabled: false


###############################
# Function Runtime Management
###############################

#### Process Runtime  ####
# Pulsar function instances are launched as processes

functionRuntimeFactoryClassName: org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory
functionRuntimeFactoryConfigs:
    # location of log files for functions
    logDirectory: logs/
    # change the jar location only when you put the java instance jar in a different location
    javaInstanceJarLocation:
    # change the python instance location only when you put the python instance jar in a different location
    pythonInstanceLocation:
    # change the extra dependencies location:
    extraFunctionDependenciesDir:

#### Thread Runtime ####
# Pulsar function instances are run as threads

#functionRuntimeFactoryClassName: org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory
#functionRuntimeFactoryConfigs:
#  # thread group name
#  threadGroupName: "Thread Function Container Group"
## Set the pulsar client memory limit
#pulsarClientMemoryLimit
#    # the max memory in bytes the pulsar client can use
#    absoluteValue:
#    # the max memory the pulsar client can use as a percentage of max direct memory set for JVM
#    percentOfMaxDirectMemory:

#### Kubernetes Runtime ####
# Pulsar function are deployed to Kubernetes

#functionRuntimeFactoryClassName: org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory
#functionRuntimeFactoryConfigs:
#    # uri to kubernetes cluster, leave it to empty and it will use the kubernetes settings in function worker
#    k8Uri:
#    # the kubernetes namespace to run the function instances. it is `default`, if this setting is left to be empty
#    jobNamespace:
#    # The Kubernetes pod name to run the function instances. It is set to
#    # `pf-<tenant>-<namespace>-<function_name>-<random_uuid(8)>` if this setting is left to be empty
#    jobName:
#    # the docker image to run function instance. by default it is `apachepulsar/pulsar`
#    pulsarDockerImageName:
#    # the docker image to run function instance according to different configurations provided by users.
#    # By default it is `apachepulsar/pulsar`.
#    # e.g:
#    # functionDockerImages:
#    #   JAVA: JAVA_IMAGE_NAME
#    #   PYTHON: PYTHON_IMAGE_NAME
#    #   GO: GO_IMAGE_NAME
#    functionDockerImages:
#    # "The image pull policy for image used to run function instance. By default it is `IfNotPresent`
#    imagePullPolicy: IfNotPresent
#    # the root directory of pulsar home directory in `pulsarDockerImageName`. by default it is `/pulsar`.
#    # if you are using your own built image in `pulsarDockerImageName`, you need to set this setting accordingly
#    pulsarRootDir:
#    # The config admin CLI allows users to customize the configuration of the admin cli tool, such as:
#    # `/bin/pulsar-admin and /bin/pulsarctl`. By default it is `/bin/pulsar-admin`. If you want to use `pulsarctl`
#    # you need to set this setting accordingly
#    configAdminCLI:
#    # this setting only takes effects if `k8Uri` is set to null. if your function worker is running as a k8 pod,
#    # setting this to true is let function worker to submit functions to the same k8s cluster as function worker
#    # is running. setting this to false if your function worker is not running as a k8 pod.
#    submittingInsidePod: false
#    # setting the pulsar service url that pulsar function should use to connect to pulsar
#    # if it is not set, it will use the pulsar service url configured in worker service
#    pulsarServiceUrl:
#    # setting the pulsar admin url that pulsar function should use to connect to pulsar
#    # if it is not set, it will use the pulsar admin url configured in worker service
#    pulsarAdminUrl:
#    # The flag indicates to install user code dependencies. (applied to python package)
#    installUserCodeDependencies:
#    # The repository that pulsar functions use to download python dependencies
#    pythonDependencyRepository:
#    # The repository that pulsar functions use to download extra python dependencies
#    pythonExtraDependencyRepository:
#    # the custom labels that function worker uses to select the nodes for pods
#    customLabels:
#    # The expected metrics collection interval, in seconds
#    expectedMetricsCollectionInterval: 30
#    # Kubernetes Runtime will periodically checkback on
#    # this configMap if defined and if there are any changes
#    # to the kubernetes specific stuff, we apply those changes
#    changeConfigMap:
#    # The namespace for storing change config map
#    changeConfigMapNamespace:
#    # The ratio cpu request and cpu limit to be set for a function/source/sink.
#    # The formula for cpu request is cpuRequest = userRequestCpu / cpuOverCommitRatio
#    cpuOverCommitRatio: 1.0
#    # The ratio memory request and memory limit to be set for a function/source/sink.
#    # The formula for memory request is memoryRequest = userRequestMemory / memoryOverCommitRatio
#    memoryOverCommitRatio: 1.0
#    # The port inside the function pod which is used by the worker to communicate with the pod
#    grpcPort: 9093
#    # The port inside the function pod on which prometheus metrics are exposed
#    metricsPort: 9094
#    # The directory inside the function pod where nar packages will be extracted
#    narExtractionDirectory:
#    # The classpath where function instance files stored
#    functionInstanceClassPath:
#    # the directory for dropping extra function dependencies
#    # if it is not an absolute path, it is relative to `pulsarRootDir`
#    extraFunctionDependenciesDir:
#    # Additional memory padding added on top of the memory requested by the function per on a per instance basis
#    percentMemoryPadding: 10
#    # The duration in seconds before the StatefulSet deleted on function stop/restart.
#    # Value must be non-negative integer. The value zero indicates delete immediately.
#    # Default is 5 seconds.
#    gracePeriodSeconds: 5

## A set of the minimum amount of resources functions must request.
## Support for this depends on function runtime.
## Only kubernetes runtime currently supports this.
# functionInstanceMinResources:
#   cpu: 1
#   ram: 1073741824
#   disk: 10737418240

## A set of the maximum amount of resources functions may request.
## Support for this depends on function runtime.
## Only kubernetes runtime currently supports this.
# functionInstanceMaxResources:
#   cpu: 16
#   ram: 17179869184
#   disk: 107374182400

## Granularities of requested resources.
## If the granularity of any type of resource is set,
## the requested resource of the type must be a multiple of the granularity.
#functionInstanceResourceGranularities:
#    cpu: 1.0
#    ram: 1073741824
#    disk: 10737418240

## If this configuration is set to be true, the amount of requested resources of all type of resources
## that have the granularity set must be the same multiples of their granularities.
#functionInstanceResourceChangeInLockStep: true

## The full class-name of an instance of RuntimeCustomizer.
## This class receives the customRuntimeOptions string and can customize details of how the runtime operates.
#runtimeCustomizerClassName: "org.apache.pulsar.functions.runtime.kubernetes.KubernetesManifestCustomizer"

## This config will pass to RuntimeCustomizer's initialize function to do initializing.
#runtimeCustomizerConfig:
#   extractLabels:
#       extraLabel: value
#   extraAnnotations:
#       extraAnnotation: value
#   nodeSelectorLabels:
#       customLabel: value
#   jobNamespace: namespace
#   tolerations:
#   - key: custom-key
#     value: value
#     effect: NoSchedule

## Config admin CLI
#configAdminCLI:

############################################
# security settings for worker service
############################################

# Enforce authentication
authenticationEnabled: false
# Enforce authorization on accessing functions api
authorizationEnabled: false
# Set of authentication provider name list, which is a list of class names
authenticationProviders:
# Authorization provider fully qualified class-name
authorizationProvider: org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider
# Set of role names that are treated as "super-user", meaning they will be able to access any admin-api
superUserRoles:

#### tls configuration for worker service
# Enable TLS
tlsEnabled: false
# Path for the TLS certificate file
tlsCertificateFilePath:
# Path for the TLS private key file
tlsKeyFilePath:
# Path for the trusted TLS certificate file
tlsTrustCertsFilePath:
# Accept untrusted TLS certificate from client
tlsAllowInsecureConnection: false
# Whether server hostname must match the common name of the certificate
tlsEnableHostnameVerification: false
# Tls cert refresh duration in seconds (set 0 to check on every new connection)
tlsCertRefreshCheckDurationSec: 300

########################
# State Management
########################

# the service url points to bookkeeper table service
# stateStorageServiceUrl: bk://localhost:4181

########################
# Connectors
########################

connectorsDirectory: ./connectors
functionsDirectory: ./functions

# Should connector config be validated during during submission
validateConnectorConfig: false

# Whether to initialize distributed log metadata by runtime.
# If it is set to true, you must ensure that it has been initialized by "bin/pulsar initialize-cluster-metadata" command.
initializedDlogMetadata: false