Distributed Scheduler CLI Changes (#1405)

* wip Signed-off-by: Cassandra Coyle <[email protected]> * rm scheduleJob logic. keep only init/uninstall logic Signed-off-by: Cassandra Coyle <[email protected]> * Fixes install and uninstall of scheduler in standalone mode. Signed-off-by: Artur Souza <[email protected]> * Fixing path for Go tools in Darwin. Signed-off-by: Artur Souza <[email protected]> * Fix Go bin location for MacOS. Signed-off-by: Artur Souza <[email protected]> * Fix min scheduler version to be 1.14.x Signed-off-by: Artur Souza <[email protected]> * Use env var to pass scheduler host. Signed-off-by: Artur Souza <[email protected]> * Fix CLI build to work with latest MacOS runners from GH Signed-off-by: Artur Souza <[email protected]> --------- Signed-off-by: Cassandra Coyle <[email protected]> Signed-off-by: Artur Souza <[email protected]> Co-authored-by: Artur Souza <[email protected]>
dapr · Jul 3, 2024 · fd1d8e8 · fd1d8e8
1 parent 4881ca1
commit fd1d8e8
Show file tree

Hide file tree

Showing 17 changed files with 279 additions and 36 deletions.
diff --git a/.github/workflows/dapr_cli.yaml b/.github/workflows/dapr_cli.yaml
@@ -39,7 +39,7 @@ jobs:
       WIX_BIN_PATH: 'C:/Program Files (x86)/WiX Toolset v3.11/bin'
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
+        os: [ubuntu-latest, windows-latest, macOS-latest, macOS-latest-large]
         target_arch: [arm, arm64, amd64]
         include:
           - os: ubuntu-latest
@@ -48,14 +48,30 @@ jobs:
             target_os: windows
           - os: macOS-latest
             target_os: darwin
+          - os: macOS-latest-large
+            target_os: darwin
         exclude:
           - os: windows-latest
             target_arch: arm
           - os: windows-latest
             target_arch: arm64
           - os: macOS-latest
             target_arch: arm
+          - os: macOS-latest
+            target_arch: amd64
+          - os: macOS-latest-large
+            target_arch: arm
+          - os: macOS-latest-large
+            target_arch: arm64
     steps:
+      - name: Prepare Go's bin location - MacOS
+        if: matrix.target_os == 'darwin'
+        run: |
+          export PATH=$HOME/bin:$PATH
+          echo "$HOME/bin" >> $GITHUB_PATH
+
+          echo "GOBIN=$HOME/bin" >> $GITHUB_ENV
+          mkdir -p $HOME/bin
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
       - name: Set up Go

diff --git a/.github/workflows/self_hosted_e2e.yaml b/.github/workflows/self_hosted_e2e.yaml
@@ -48,20 +48,29 @@ jobs:
       # TODO: Remove this when our E2E tests are stable for podman on MacOS.
       fail-fast: false # Keep running if one leg fails.
       matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
+        # See https://github.com/actions/runner-images
+        os: [macos-latest-large, ubuntu-latest, windows-latest]
         target_arch: [amd64]
         dapr_install_mode: [slim, complete]
         include:
           - os: ubuntu-latest
             target_os: linux
-          - os: macOS-latest
+          - os: macos-latest-large
             target_os: darwin
           - os: windows-latest
             target_os: windows
         exclude:
           - os: windows-latest
             dapr_install_mode: complete
     steps:
+      - name: Prepare Go's bin location - MacOS
+        if: matrix.os == 'macos-latest-large'
+        run: |
+          export PATH=$HOME/bin:$PATH
+          echo "$HOME/bin" >> $GITHUB_PATH
+
+          echo "GOBIN=$HOME/bin" >> $GITHUB_ENV
+          mkdir -p $HOME/bin
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
       - name: Set up Go
@@ -101,7 +110,7 @@ jobs:
             ${{ matrix.target_os }}-${{ matrix.target_arch }}-go-${{ steps.setup-go.outputs.go-version }}-
       - name: Install podman - MacOS
         timeout-minutes: 15
-        if: matrix.os == 'macos-latest' && matrix.dapr_install_mode == 'complete'
+        if: matrix.os == 'macos-latest-large' && matrix.dapr_install_mode == 'complete'
         run: |
           # Install podman
           curl -sL -o podman.pkg https://github.com/containers/podman/releases/download/v${{ env.PODMAN_VERSION }}/podman-installer-macos-amd64.pkg
@@ -149,7 +158,7 @@ jobs:
           echo "DAPR_DASHBOARD_LATEST_STABLE_VERSION=$LATEST_STABLE_DASHBOARD_VERSION" >> $GITHUB_ENV
         shell: bash
       - name: Set the test timeout - MacOS
-        if: matrix.os == 'macos-latest'
+        if: matrix.os == 'macos-latest-large'
         run: echo "E2E_SH_TEST_TIMEOUT=30m" >> $GITHUB_ENV
       - name: Run E2E tests with GHCR
         # runs every 6hrs

diff --git a/.gitignore b/.gitignore
@@ -6,6 +6,9 @@
 *.dylib
 cli
 
+# Handy directory to keep local scripts and help files.
+.local/
+
 # Mac's metadata folder
 .DS_Store
 
@@ -37,4 +40,4 @@ go.work
 #Wix files
 *.wixobj
 *.wixpdb
-*.msi
+*.msi
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -40,7 +40,7 @@ Before you file an issue, make sure you've checked the following:
         - 👎 down-vote
 1. For bugs
     - Check it's not an environment issue. For example, if running on Kubernetes, make sure prerequisites are in place. (state stores, bindings, etc.)
-    - You have as much data as possible. This usually comes in the form of logs and/or stacktrace. If running on Kubernetes or other environment, look at the logs of the Dapr services (runtime, operator, placement service). More details on how to get logs can be found [here](https://docs.dapr.io/operations/troubleshooting/logs-troubleshooting/).
+    - You have as much data as possible. This usually comes in the form of logs and/or stacktrace. If running on Kubernetes or other environment, look at the logs of the Dapr services (runtime, operator, placement, scheduler service). More details on how to get logs can be found [here](https://docs.dapr.io/operations/troubleshooting/logs-troubleshooting/).
 1. For proposals
     - Many changes to the Dapr runtime may require changes to the API. In that case, the best place to discuss the potential feature is the main [Dapr repo](https://github.com/dapr/dapr).
     - Other examples could include bindings, state stores or entirely new components.

diff --git a/Makefile b/Makefile
@@ -59,6 +59,7 @@ ifeq ($(LOCAL_OS),Linux)
 else ifeq ($(LOCAL_OS),Darwin)
    TARGET_OS_LOCAL = darwin
    GOLANGCI_LINT:=golangci-lint
+   PATH := $(PATH):$(HOME)/go/bin/darwin_$(GOARCH)
    export ARCHIVE_EXT = .tar.gz
 else
    TARGET_OS_LOCAL ?= windows

diff --git a/README.md b/README.md
@@ -68,7 +68,7 @@ Install windows Dapr CLI using MSI package.
 
 ### Install Dapr on your local machine (self-hosted)
 
-In self-hosted mode, dapr can be initialized using the CLI  with the placement, redis and zipkin containers enabled by default(recommended) or without them which also does not require docker to be available in the environment.
+In self-hosted mode, dapr can be initialized using the CLI  with the placement, scheduler, redis, and zipkin containers enabled by default(recommended) or without them which also does not require docker to be available in the environment.
 
 #### Initialize Dapr
 
@@ -89,6 +89,7 @@ Output should look like so:
 ✅  Downloaded binaries and completed components set up.
 ℹ️  daprd binary has been installed to  $HOME/.dapr/bin.
 ℹ️  dapr_placement container is running.
+ℹ️  dapr_scheduler container is running.
 ℹ️  dapr_redis container is running.
 ℹ️  dapr_zipkin container is running.
 ℹ️  Use `docker ps` to check running containers.
@@ -118,10 +119,11 @@ Output should look like so:
 ✅  Downloaded binaries and completed components set up.
 ℹ️  daprd binary has been installed to $HOME/.dapr/bin.
 ℹ️  placement binary has been installed.
+ℹ️  scheduler binary has been installed.
 ✅  Success! Dapr is up and running. To get started, go here: https://aka.ms/dapr-getting-started
 ```
 
->Note: When initializing Dapr with the `--slim` flag only the Dapr runtime binary and the placement service binary are installed. An empty default components folder is created with no default configuration files. During `dapr run` user should use `--resources-path` (`--components-path` is deprecated and will be removed in future releases) to point to a components directory with custom configurations files or alternatively place these files in the default directory. For Linux/MacOS, the default components directory path is `$HOME/.dapr/components` and for Windows it is `%USERPROFILE%\.dapr\components`.
+>Note: When initializing Dapr with the `--slim` flag only the Dapr runtime, placement, and scheduler service binaries are installed. An empty default components folder is created with no default configuration files. During `dapr run` user should use `--resources-path` (`--components-path` is deprecated and will be removed in future releases) to point to a components directory with custom configurations files or alternatively place these files in the default directory. For Linux/MacOS, the default components directory path is `$HOME/.dapr/components` and for Windows it is `%USERPROFILE%\.dapr\components`.
 
 #### Install a specific runtime version
 
@@ -171,7 +173,7 @@ Move to the bundle directory and run the following command:
 
 > If you are not running the above command from the bundle directory, provide the full path to bundle directory as input. For example, assuming the bundle directory path is $HOME/daprbundle, run `$HOME/daprbundle/dapr init --from-dir $HOME/daprbundle` to have the same behavior.
 
-> Note: Dapr Installer bundle just contains the placement container apart from the binaries and so `zipkin` and `redis` are not enabled by default. You can pull the images locally either from network or private registry and run as follows:
+> Note: Dapr Installer bundle just contains the placement and scheduler containers apart from the binaries and so `zipkin` and `redis` are not enabled by default. You can pull the images locally either from network or private registry and run as follows:
 
 ```bash
 docker run --name "dapr_zipkin" --restart always -d -p 9411:9411 openzipkin/zipkin
@@ -199,6 +201,9 @@ dapr init --network dapr-network
 > Note: When installed to a specific Docker network, you will need to add the `--placement-host-address` arguments to `dapr run` commands run in any containers within that network.
 > The format of `--placement-host-address` argument is either `<hostname>` or `<hostname>:<port>`. If the port is omitted, the default port `6050` for Windows and `50005` for Linux/MacOS applies.
 
+> Note: When installed to a specific Docker network, you will need to add the `--scheduler-host-address` arguments to `dapr run` commands run in any containers within that network.
+> The format of `--scheduler-host-address` argument is either `<hostname>` or `<hostname>:<port>`. If the port is omitted, the default port `6060` for Windows and `50006` for Linux/MacOS applies.
+
 #### Install with a specific container runtime
 
 You can install the Dapr runtime using a specific container runtime
@@ -228,7 +233,7 @@ For more details, see the docs for dev containers with [Visual Studio Code](http
 
 ### Uninstall Dapr in a standalone mode
 
-Uninstalling will remove daprd binary and the placement container (if installed with Docker or the placement binary if not).
+Uninstalling will remove daprd binary along with the placement and scheduler containers (if installed with Docker or the placement and scheduler binaries if not).
 
 
 ```bash
@@ -237,15 +242,15 @@ dapr uninstall
 
 > For Linux users, if you run your docker cmds with sudo, you need to use "**sudo dapr uninstall**" to remove the containers.
 
-The command above won't remove the redis or zipkin containers by default in case you were using it for other purposes.  It will also not remove the default dapr folder that was created on `dapr init`. To remove all the containers (placement, redis, zipkin) and also the default dapr folder created on init run:
+The command above won't remove the redis or zipkin containers by default in case you were using it for other purposes.  It will also not remove the default dapr folder that was created on `dapr init`. To remove all the containers (placement, scheduler, redis, zipkin) and also the default dapr folder created on init run:
 
 ```bash
 dapr uninstall --all
 ```
 
 The above command can also be run when Dapr has been installed in a non-docker environment, it will only remove the installed binaries and the default dapr folder in that case.
 
-> NB: The `dapr uninstall` command will always try to remove the placement binary/service and will throw an error is not able to.
+> NB: The `dapr uninstall` command will always try to remove the placement and scheduler binaries/services and will throw an error is not able to.
 
 **You should always run a `dapr uninstall` before running another `dapr init`.**
 
@@ -407,7 +412,7 @@ dapr init --network dapr-network
 dapr run --app-id nodeapp --placement-host-address dapr_placement node app.js
 ```
 
-> Note: When in a specific Docker network, the Redis, Zipkin and placement service containers are given specific network aliases, `dapr_redis`, `dapr_zipkin` and `dapr_placement`, respectively. The default configuration files reflect the network alias rather than `localhost` when a docker network is specified.
+> Note: When in a specific Docker network, the Redis, Zipkin and placement and scheduler service containers are given specific network aliases, `dapr_redis`, `dapr_zipkin`, `dapr_placement`, and `dapr_scheduler`, respectively. The default configuration files reflect the network alias rather than `localhost` when a docker network is specified.
 
 ### Use gRPC
 

diff --git a/cmd/init.go b/cmd/init.go
@@ -210,7 +210,7 @@ func init() {
 	InitCmd.Flags().BoolVarP(&devMode, "dev", "", false, "Use Dev mode. Deploy Redis, Zipkin also in the Kubernetes cluster")
 	InitCmd.Flags().BoolVarP(&wait, "wait", "", false, "Wait for Kubernetes initialization to complete")
 	InitCmd.Flags().UintVarP(&timeout, "timeout", "", 300, "The wait timeout for the Kubernetes installation")
-	InitCmd.Flags().BoolVarP(&slimMode, "slim", "s", false, "Exclude placement service, Redis and Zipkin containers from self-hosted installation")
+	InitCmd.Flags().BoolVarP(&slimMode, "slim", "s", false, "Exclude placement service, scheduler service, Redis and Zipkin containers from self-hosted installation")
 	InitCmd.Flags().StringVarP(&runtimeVersion, "runtime-version", "", defaultRuntimeVersion, "The version of the Dapr runtime to install, for example: 1.0.0")
 	InitCmd.Flags().StringVarP(&dashboardVersion, "dashboard-version", "", defaultDashboardVersion, "The version of the Dapr dashboard to install, for example: 0.13.0")
 	InitCmd.Flags().StringVarP(&initNamespace, "namespace", "n", "dapr-system", "The Kubernetes namespace to install Dapr in")

diff --git a/cmd/run.go b/cmd/run.go
@@ -120,6 +120,7 @@ dapr run --run-file /path/to/directory -k
 	Args: cobra.MinimumNArgs(0),
 	PreRun: func(cmd *cobra.Command, args []string) {
 		viper.BindPFlag("placement-host-address", cmd.Flags().Lookup("placement-host-address"))
+		viper.BindPFlag("scheduler-host-address", cmd.Flags().Lookup("scheduler-host-address"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		if len(runFilePath) > 0 {
@@ -171,6 +172,7 @@ dapr run --run-file /path/to/directory -k
 			MaxConcurrency:     maxConcurrency,
 			AppProtocol:        protocol,
 			PlacementHostAddr:  viper.GetString("placement-host-address"),
+			SchedulerHostAddr:  viper.GetString("scheduler-host-address"),
 			ComponentsPath:     componentsPath,
 			ResourcesPaths:     resourcesPaths,
 			AppSSL:             appSSL,
@@ -454,6 +456,7 @@ func init() {
 	// By marking this as deprecated, the flag will be hidden from the help menu, but will continue to work. It will show a warning message when used.
 	RunCmd.Flags().MarkDeprecated("components-path", "This flag is deprecated and will be removed in the future releases. Use \"resources-path\" flag instead")
 	RunCmd.Flags().String("placement-host-address", "localhost", "The address of the placement service. Format is either <hostname> for default port or <hostname>:<port> for custom port")
+	RunCmd.Flags().String("scheduler-host-address", "localhost", "The address of the scheduler service. Format is either <hostname> for default port or <hostname>:<port> for custom port")
 	// TODO: Remove below flag once the flag is removed in runtime in future release.
 	RunCmd.Flags().BoolVar(&appSSL, "app-ssl", false, "Enable https when Dapr invokes the application")
 	RunCmd.Flags().MarkDeprecated("app-ssl", "This flag is deprecated and will be removed in the future releases. Use \"app-protocol\" flag with https or grpcs values instead")

diff --git a/cmd/uninstall.go b/cmd/uninstall.go
@@ -43,7 +43,7 @@ var UninstallCmd = &cobra.Command{
 # Uninstall from self-hosted mode
 dapr uninstall
 
-# Uninstall from self-hosted mode and remove .dapr directory, Redis, Placement and Zipkin containers
+# Uninstall from self-hosted mode and remove .dapr directory, Redis, Placement, Scheduler, and Zipkin containers
 dapr uninstall --all
 
 # Uninstall from Kubernetes
@@ -99,7 +99,7 @@ func init() {
 	UninstallCmd.Flags().BoolVarP(&uninstallKubernetes, "kubernetes", "k", false, "Uninstall Dapr from a Kubernetes cluster")
 	UninstallCmd.Flags().BoolVarP(&uninstallDev, "dev", "", false, "Uninstall Dapr Redis and Zipking installations from Kubernetes cluster")
 	UninstallCmd.Flags().UintVarP(&timeout, "timeout", "", 300, "The timeout for the Kubernetes uninstall")
-	UninstallCmd.Flags().BoolVar(&uninstallAll, "all", false, "Remove .dapr directory, Redis, Placement and Zipkin containers on local machine, and CRDs on a Kubernetes cluster")
+	UninstallCmd.Flags().BoolVar(&uninstallAll, "all", false, "Remove .dapr directory, Redis, Placement, Scheduler, and Zipkin containers on local machine, and CRDs on a Kubernetes cluster")
 	UninstallCmd.Flags().String("network", "", "The Docker network from which to remove the Dapr runtime")
 	UninstallCmd.Flags().StringVarP(&uninstallNamespace, "namespace", "n", "dapr-system", "The Kubernetes namespace to uninstall Dapr from")
 	UninstallCmd.Flags().BoolP("help", "h", false, "Print this help message")

diff --git a/go.mod b/go.mod
@@ -79,6 +79,7 @@ require (
 	github.com/BurntSushi/toml v1.2.1 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver v1.5.0
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
 	github.com/Masterminds/squirrel v1.5.3 // indirect
 	github.com/Microsoft/go-winio v0.6.0 // indirect

diff --git a/go.sum b/go.sum
@@ -89,6 +89,8 @@ github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6
 github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
 github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=

diff --git a/pkg/standalone/bundle.go b/pkg/standalone/bundle.go
@@ -55,10 +55,10 @@ func isStringNilOrEmpty(val *string) bool {
 	return val == nil || strings.TrimSpace(*val) == ""
 }
 
-func (b *bundleDetails) getPlacementImageName() string {
+func (b *bundleDetails) getDaprImageName() string {
 	return *b.DaprImageName
 }
 
-func (b *bundleDetails) getPlacementImageFileName() string {
+func (b *bundleDetails) getDaprImageFileName() string {
 	return *b.DaprImageFileName
 }
diff --git a/pkg/standalone/bundle_test.go b/pkg/standalone/bundle_test.go
@@ -44,8 +44,8 @@ func TestParseDetails(t *testing.T) {
 	assert.Equal(t, "0.10.0", *bd.DashboardVersion, "expected versions to match")
 	assert.Equal(t, "dist", *bd.BinarySubDir, "expected value to match")
 	assert.Equal(t, "docker", *bd.ImageSubDir, "expected value to match")
-	assert.Equal(t, "daprio/dapr:1.7.2", bd.getPlacementImageName(), "expected value to match")
-	assert.Equal(t, "daprio-dapr-1.7.2.tar.gz", bd.getPlacementImageFileName(), "expected value to match")
+	assert.Equal(t, "daprio/dapr:1.7.2", bd.getDaprImageName(), "expected value to match")
+	assert.Equal(t, "daprio-dapr-1.7.2.tar.gz", bd.getDaprImageFileName(), "expected value to match")
 }
 
 func TestParseDetailsMissingDetails(t *testing.T) {

diff --git a/pkg/standalone/common.go b/pkg/standalone/common.go
@@ -17,6 +17,7 @@ import (
 	"os"
 	path_filepath "path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 )
 
@@ -25,8 +26,10 @@ const (
 	DefaultConfigFileName   = "config.yaml"
 	DefaultResourcesDirName = "resources"
 
-	defaultDaprBinDirName    = "bin"
-	defaultComponentsDirName = "components"
+	defaultDaprBinDirName       = "bin"
+	defaultComponentsDirName    = "components"
+	defaultSchedulerDirName     = "scheduler"
+	defaultSchedulerDataDirName = "data"
 )
 
 // GetDaprRuntimePath returns the dapr runtime installation path.
@@ -58,6 +61,16 @@ func getDaprBinPath(daprDir string) string {
 	return path_filepath.Join(daprDir, defaultDaprBinDirName)
 }
 
+// getSchedulerDataPath returns the data path of a given instance
+// Receiving instanceID allows multiple instances of scheduler to run locally in the future.
+func getSchedulerDataPath(daprDir string, instanceID int) string {
+	return path_filepath.Join(
+		daprDir,
+		defaultSchedulerDirName,
+		defaultSchedulerDataDirName,
+		strconv.Itoa(instanceID))
+}
+
 func binaryFilePathWithDir(binaryDir string, binaryFilePrefix string) string {
 	binaryPath := path_filepath.Join(binaryDir, binaryFilePrefix)
 	if runtime.GOOS == daprWindowsOS {